mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 12:19:13 -05:00
feat: round 3 reference and data
This commit is contained in:
Abdessamad Derraz committed
commit
d168ceb8a8
18 files changed
+1287
-101
No files matched your search
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/database.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/database.schema.json",
|
||||
"title": "RetroBIOS content database",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/emulator.schema.json",
|
||||
"title": "Emulator BIOS Profile",
|
||||
"description": "One YAML per emulator core or engine, documenting every external file the code loads. Field reference: https://abdess.github.io/retrobios/wiki/profiling/",
|
||||
"type": "object",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/install-manifest.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-manifest.schema.json",
|
||||
"title": "RetroBIOS installer manifest",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
@@ -54,21 +54,26 @@
|
||||
},
|
||||
"platform": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
"minLength": 1,
|
||||
"description": "Registry key of the platform (batocera)."
|
||||
},
|
||||
"display_name": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
"minLength": 1,
|
||||
"description": "Name the site shows for the platform."
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Version of the platform's own list this manifest was built from."
|
||||
},
|
||||
"generated": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
"format": "date-time",
|
||||
"description": "When the manifest was written."
|
||||
},
|
||||
"base_destination": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "The BIOS folder entries are written below, as the platform names it. Empty when entries start one level higher (RetroDECK)."
|
||||
},
|
||||
"detect": {
|
||||
"type": "array",
|
||||
@@ -84,11 +89,13 @@
|
||||
},
|
||||
"total_files": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
"minimum": 0,
|
||||
"description": "Number of entries in files."
|
||||
},
|
||||
"total_size": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
"minimum": 0,
|
||||
"description": "Bytes of the files in files."
|
||||
},
|
||||
"pack_files": {
|
||||
"description": "Files the platform's pack holds once extracted: this list, the data directories and the pack's own documents.",
|
||||
@@ -102,7 +109,8 @@
|
||||
},
|
||||
"total_omitted": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
"minimum": 0,
|
||||
"description": "Number of entries in omitted_files."
|
||||
},
|
||||
"core_names": {
|
||||
"description": "Each core a file names, with the other names it goes by upstream (the buildbot's mednafen_psx for beetle_psx).",
|
||||
@@ -331,13 +339,15 @@
|
||||
"pattern": "^[^/\\\\]+$"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Files the pack needs and the repository cannot serve, with the reason."
|
||||
},
|
||||
"files": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/file"
|
||||
}
|
||||
},
|
||||
"description": "Every file the installer fetches, one entry per destination."
|
||||
},
|
||||
"case_variants": {
|
||||
"description": "Files the folded list serves under another case, installed only where the target directory keeps case.",
|
||||
@@ -459,22 +469,27 @@
|
||||
"properties": {
|
||||
"dest": {
|
||||
"type": "string",
|
||||
"pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"
|
||||
"pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$",
|
||||
"description": "Path below base_destination, or below the save directory when root is save."
|
||||
},
|
||||
"sha1": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-f0-9]{40}$"
|
||||
"pattern": "^[a-f0-9]{40}$",
|
||||
"description": "SHA1 of the copy the repository serves. The installer checks it after download."
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-f0-9]{64}$"
|
||||
"pattern": "^[a-f0-9]{64}$",
|
||||
"description": "SHA256 of the same copy."
|
||||
},
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
"minimum": 0,
|
||||
"description": "Bytes of the same copy."
|
||||
},
|
||||
"repo_path": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Path in the repository. Fetched from https://raw.githubusercontent.com/Abdess/retrobios/<ref>/<repo_path>, ref being main unless RETROBIOS_REF names a tag."
|
||||
},
|
||||
"cores": {
|
||||
"oneOf": [
|
||||
@@ -488,14 +503,17 @@
|
||||
},
|
||||
"uniqueItems": true
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Profiles whose core brings the file into the pack. Null for a file the platform's own list declares."
|
||||
},
|
||||
"storage": {
|
||||
"const": "release"
|
||||
"const": "release",
|
||||
"description": "release when the file is a large-files release asset."
|
||||
},
|
||||
"release_asset": {
|
||||
"type": "string",
|
||||
"pattern": "^[^/\\\\]+$"
|
||||
"pattern": "^[^/\\\\]+$",
|
||||
"description": "Asset of the large-files release, fetched from https://github.com/Abdess/retrobios/releases/download/large-files/<asset> instead of repo_path."
|
||||
},
|
||||
"systems": {
|
||||
"description": "Every platform system that declares the file, for the installer's --system.",
|
||||
@@ -929,5 +947,6 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "What a platform's pack installs, one file per entry, as install.py reads it. The platform's list says what the setup checks, this list says what the pack writes: the list's files plus every file the setup's cores load."
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/pack-manifest.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/pack-manifest.schema.json",
|
||||
"title": "RetroBIOS pack integrity manifest",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/platform.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/platform.schema.json",
|
||||
"title": "Platform Configuration",
|
||||
"description": "Schema for retrogaming platform BIOS configuration files",
|
||||
"type": "object",
|
||||
@@ -506,6 +506,14 @@
|
||||
"type": "boolean",
|
||||
"description": "Added by our scraper from the cores' source, not declared in the platform's own file"
|
||||
},
|
||||
"shared_group": {
|
||||
"type": "string",
|
||||
"description": "Set on load, never written in a platform file: the entry joined the system from this group of _shared.yml, named by the system's includes. The platform's own file does not carry it"
|
||||
},
|
||||
"zip_digest": {
|
||||
"const": "composite",
|
||||
"description": "Set on load, never written in a platform file: the md5 of this ZIP is the md5 of its members concatenated in name order, as Recalbox computes it, never the md5 of the archive bytes"
|
||||
},
|
||||
"native_name": {
|
||||
"type": "string",
|
||||
"description": "Display name the platform gives its own system"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/release.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/release.schema.json",
|
||||
"title": "RetroBIOS release record",
|
||||
"description": "What the packs of the latest release hold, read from the archives when the release is cut.",
|
||||
"type": "object",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/site-api-envelope.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/site-api-envelope.schema.json",
|
||||
"title": "RetroBIOS static API v1 envelope",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
@@ -22,7 +22,11 @@
|
||||
"commit": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$",
|
||||
"description": "The repository commit the data was read at."
|
||||
"description": "The repository commit the data was read at. Every envelope carries it when the site is built from a git checkout."
|
||||
},
|
||||
"license": {
|
||||
"type": "string",
|
||||
"description": "The terms the metadata is offered under, from LICENSE. They never cover BIOS or firmware files (NOTICE)."
|
||||
},
|
||||
"kind": {
|
||||
"type": "string",
|
||||
@@ -33,7 +37,10 @@
|
||||
"systems",
|
||||
"verification-and-coverage-gaps",
|
||||
"file-locations",
|
||||
"file-lookup"
|
||||
"file-lookup",
|
||||
"index",
|
||||
"system-index",
|
||||
"core-index"
|
||||
]
|
||||
},
|
||||
"count": {
|
||||
@@ -115,7 +122,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"profile": {
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"description": "The profile as its YAML states it, valid against emulator.schema.json (https://abdess.github.io/retrobios/api/v1/schemas/emulator.schema.json)."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,10 +158,15 @@
|
||||
"type": "string"
|
||||
},
|
||||
"contract": {
|
||||
"type": "object"
|
||||
"type": "object",
|
||||
"description": "The platform's list as the tools load it, valid against platform.schema.json (https://abdess.github.io/retrobios/api/v1/schemas/platform.schema.json). Entries a shared group adds carry shared_group, and a Recalbox ZIP entry carries zip_digest."
|
||||
},
|
||||
"coverage": {
|
||||
"type": "object"
|
||||
},
|
||||
"registry": {
|
||||
"type": "object",
|
||||
"description": "The platform's registry entry beside its list: install detection, logo, scraper and schedule. Not part of the list."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -261,7 +274,8 @@
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Site-absolute path of the file's card, starting at /retrobios/."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -384,7 +398,8 @@
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "Site-absolute path of the page that shows the value, starting at /retrobios/. Prefix https://abdess.github.io for a full URL."
|
||||
},
|
||||
"sha1": {
|
||||
"type": "string",
|
||||
@@ -392,6 +407,133 @@
|
||||
},
|
||||
"owner": {
|
||||
"type": "string"
|
||||
},
|
||||
"system_name": {
|
||||
"type": "string",
|
||||
"description": "The name the site gives the system the record belongs to (Sony PlayStation): the console a collection file is filed under, the system a platform lists it in, the system of an emulator's entry. Absent when none is known."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "index"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"items": {
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "Relative to this document's URL."
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "system-index"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"items": {
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"name",
|
||||
"manufacturer",
|
||||
"url",
|
||||
"sha1"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "The system id a platform list or an emulator profile writes."
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "The name the site shows for the id."
|
||||
},
|
||||
"manufacturer": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"description": "Site-absolute path of the console section holding the id's files, null when no section holds them."
|
||||
},
|
||||
"sha1": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"description": "SHA1 of each collection file a platform list or an emulator profile names for the id."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "core-index"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"items": {
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"profile"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "A core name as the buildbot or a platform writes it (mednafen_psx), a profile id, or an alias profile."
|
||||
},
|
||||
"profile": {
|
||||
"type": "string",
|
||||
"description": "The profile id whose api/v1/emulators/<id>.json and page describe it."
|
||||
},
|
||||
"preferred": {
|
||||
"type": "boolean",
|
||||
"description": "The profile a platform list naming this core gets: its own key first, else the first claimant."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/stats.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/stats.schema.json",
|
||||
"title": "RetroBIOS public statistics",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/Abdess/retrobios/schemas/target-manifest.schema.json",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/target-manifest.schema.json",
|
||||
"title": "RetroBIOS hardware target manifest",
|
||||
"description": "Per-target core availability. A null core list means the target publishes no inventory: a known target with no filter, not an absent one.",
|
||||
"type": "object",
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Check a BIOS folder against one platform's list, from the published data only.
|
||||
|
||||
python check_platform_folder.py batocera path/to/bios
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
API = "https://abdess.github.io/retrobios/api/v1/platforms/"
|
||||
|
||||
|
||||
def accepts(entry: dict, path: Path, mode: str) -> bool:
|
||||
"""Whether the platform's own check passes the file at path."""
|
||||
raw = entry.get(mode) or ""
|
||||
values = raw if isinstance(raw, list) else str(raw).split(",")
|
||||
declared = [value.strip().lower() for value in values if value.strip()]
|
||||
if mode == "existence" or not declared:
|
||||
return path.is_file() # nothing to compare: the platform checks presence
|
||||
try:
|
||||
if entry.get("zipped_file"): # the hash of one member, its name casefolded
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = {name.casefold(): name for name in archive.namelist()}
|
||||
member = names.get(entry["zipped_file"].casefold())
|
||||
return bool(member) and hashlib.new(mode, archive.read(member)).hexdigest() in declared
|
||||
actual = hashlib.new(mode, path.read_bytes()).hexdigest()
|
||||
if entry.get("zip_digest") != "composite" and any(actual.startswith(d) for d in declared):
|
||||
return True # a truncated md5 (Batocera) compares by its prefix
|
||||
if mode == "md5" and path.suffix.lower() == ".zip": # Recalbox: members in name order
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = sorted(name for name in archive.namelist() if not name.endswith("/"))
|
||||
return hashlib.md5(b"".join(map(archive.read, names))).hexdigest() in declared
|
||||
except (OSError, zipfile.BadZipFile):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def check(platform: str, root: Path, api: str = API) -> list[tuple[str, str]]:
|
||||
"""(destination, ok | wrong | missing) for each entry of the list."""
|
||||
with urllib.request.urlopen(f"{api}{platform}.json") as response:
|
||||
contract = json.load(response)["items"][0]["contract"]
|
||||
mode = contract["verification_mode"]
|
||||
results = []
|
||||
for system in contract["systems"].values():
|
||||
for entry in system.get("files") or []:
|
||||
destination = entry.get("destination") or entry["name"]
|
||||
path = root / destination
|
||||
verdict = "ok" if accepts(entry, path, mode) else "wrong" if path.is_file() else "missing"
|
||||
results.append((destination, verdict))
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
found = check(sys.argv[1], Path(sys.argv[2]))
|
||||
for verdict in ("ok", "wrong", "missing"):
|
||||
names = sorted({dest for dest, seen in found if seen == verdict})
|
||||
print(f"{verdict}: {len(names)}" + "".join(f"\n {name}" for name in names if verdict != "ok"))
|
||||
+465
-56
@@ -111,6 +111,9 @@ STARLIGHT_CONTENT = "src/content/docs"
|
||||
SITE_NAME = "RetroBIOS"
|
||||
REPO_URL = "https://github.com/Abdess/retrobios"
|
||||
RELEASE_URL = f"{REPO_URL}/releases/latest"
|
||||
# What LICENSE grants over the published data: its MIT terms name the
|
||||
# metadata. The BIOS files themselves are under NOTICE, never this.
|
||||
DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)"
|
||||
SITE_URL = "https://abdess.github.io/retrobios/"
|
||||
GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"]
|
||||
# Generated directories of files served as they are, not pages.
|
||||
@@ -849,6 +852,36 @@ def _sha1_by_path(db: dict | None) -> dict[str, str]:
|
||||
return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()}
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _contract_keys() -> frozenset[str]:
|
||||
"""The top-level keys platform.schema.json allows in a platform's list."""
|
||||
schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8"))
|
||||
return frozenset(schema.get("properties") or {})
|
||||
|
||||
|
||||
def _split_contract(config: dict) -> tuple[dict, dict]:
|
||||
"""The list, valid against platform.schema.json, and the registry entry.
|
||||
|
||||
load_platform_config merges the registry into the list: `install`,
|
||||
`logo`, `scraper` and `_key` made all 18 contracts fail the published
|
||||
schema. They stay published, beside the contract.
|
||||
"""
|
||||
allowed = _contract_keys()
|
||||
contract = {key: value for key, value in config.items() if key in allowed}
|
||||
registry = {key: value for key, value in config.items()
|
||||
if key not in allowed and key != "_key"}
|
||||
return contract, registry
|
||||
|
||||
|
||||
# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP
|
||||
# member's when zipped_file is set, and a CSV without it judged 106 good
|
||||
# Batocera files wrong.
|
||||
ENTRY_EXPORT_FIELDS = (
|
||||
"zipped_file", "hash_match_mandatory", "mandatory_declared", "curated",
|
||||
"shared_group", "mirror_of", "zip_digest",
|
||||
)
|
||||
|
||||
|
||||
def _platform_export_rows(
|
||||
coverages: dict, db: dict | None = None,
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
@@ -859,6 +892,7 @@ def _platform_export_rows(
|
||||
for key, coverage in sorted(coverages.items()):
|
||||
config = coverage["config"]
|
||||
outcome = _verify_outcomes(coverage, sha1_by_path)
|
||||
contract, registry = _split_contract(config)
|
||||
items.append({
|
||||
"id": key,
|
||||
"name": coverage["platform"],
|
||||
@@ -869,7 +903,8 @@ def _platform_export_rows(
|
||||
"core_present", "core_missing", "core_unsourceable",
|
||||
)
|
||||
},
|
||||
"contract": config,
|
||||
"contract": contract,
|
||||
"registry": registry,
|
||||
})
|
||||
for system_id, system in sorted(config.get("systems", {}).items()):
|
||||
for entry in system.get("files", []) or []:
|
||||
@@ -889,11 +924,20 @@ def _platform_export_rows(
|
||||
"sha256": _json_text(entry.get("sha256")),
|
||||
"md5": _json_text(entry.get("md5")),
|
||||
"crc32": _json_text(entry.get("crc32")),
|
||||
**{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS},
|
||||
**outcome(system_id, entry.get("name", "")),
|
||||
})
|
||||
return items, file_rows
|
||||
|
||||
|
||||
def _export_flag(value: object) -> object:
|
||||
"""A per-entry field as CSV and SQLite hold it: booleans stay booleans,
|
||||
an absent field is empty."""
|
||||
if value is None:
|
||||
return ""
|
||||
return value if isinstance(value, bool) else str(value)
|
||||
|
||||
|
||||
def _emulator_export_items(profiles: dict) -> list[dict]:
|
||||
return [
|
||||
{"id": key, "profile": profile}
|
||||
@@ -1132,7 +1176,9 @@ def _write_sqlite_export(
|
||||
destination TEXT NOT NULL, required INTEGER NOT NULL,
|
||||
region TEXT, variant_group TEXT, size INTEGER,
|
||||
sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT,
|
||||
status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT
|
||||
status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT,
|
||||
zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER,
|
||||
curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT
|
||||
);
|
||||
CREATE TABLE emulators (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT,
|
||||
@@ -1177,6 +1223,7 @@ def _write_sqlite_export(
|
||||
**({"commit": commit} if commit else {}),
|
||||
"source": REPO_URL,
|
||||
"scope": "metadata only, no BIOS or firmware payload bytes",
|
||||
"license": DATA_LICENSE,
|
||||
}
|
||||
connection.executemany(
|
||||
"INSERT INTO metadata(key, value) VALUES (?, ?)",
|
||||
@@ -1211,7 +1258,8 @@ def _write_sqlite_export(
|
||||
"INSERT INTO platform_files VALUES "
|
||||
"(:platform_id, :system, :name, :destination, :required, :region, "
|
||||
":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
|
||||
":collection_path, :resolved_sha1)",
|
||||
":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, "
|
||||
":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)",
|
||||
platform_files,
|
||||
)
|
||||
for item in emulator_items:
|
||||
@@ -1402,13 +1450,21 @@ def generate_data_exports(
|
||||
json.dumps(db, ensure_ascii=False, indent=2) + "\n",
|
||||
)
|
||||
|
||||
schema_names = (
|
||||
"database.schema.json", "emulator.schema.json", "platform.schema.json",
|
||||
"site-api-envelope.schema.json", "stats.schema.json",
|
||||
)
|
||||
for schema_name in schema_names:
|
||||
shutil.copy2(Path("schemas") / schema_name, schemas_dest / schema_name)
|
||||
_record(schemas_dest / schema_name)
|
||||
# Every schema the published files are written against, the manifests'
|
||||
# too: an integrator had the platform's list and not what the pack writes.
|
||||
for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")):
|
||||
shutil.copy2(schema_path, schemas_dest / schema_path.name)
|
||||
_record(schemas_dest / schema_path.name)
|
||||
|
||||
manifest_assets = _publish_manifests(api)
|
||||
for filename, (kind, items) in {
|
||||
"systems.json": ("system-index", system_index_items(db, coverages, profiles)),
|
||||
"cores.json": ("core-index", core_index_items(profiles)),
|
||||
}.items():
|
||||
write_if_changed(str(api / filename), json.dumps({
|
||||
"schema_version": 1, "generated_at": generated_at, **stamp,
|
||||
"kind": kind, "count": len(items), "items": items,
|
||||
}, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
file_rows = []
|
||||
for sha1, entry in sorted(db.get("files", {}).items()):
|
||||
@@ -1443,7 +1499,7 @@ def generate_data_exports(
|
||||
"platform_id", "platform", "system", "name", "destination",
|
||||
"required", "region", "variant_group", "size", "sha1",
|
||||
"sha256", "md5", "crc32", "status", "category",
|
||||
"collection_path", "resolved_sha1",
|
||||
"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,
|
||||
],
|
||||
platform_files,
|
||||
),
|
||||
@@ -1495,6 +1551,11 @@ def generate_data_exports(
|
||||
(api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"),
|
||||
(api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"),
|
||||
(api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"),
|
||||
(api / "systems.json", "System ids, their names and sections", "application/json",
|
||||
"schemas/site-api-envelope.schema.json"),
|
||||
(api / "cores.json", "Core names and the profile each one is", "application/json",
|
||||
"schemas/site-api-envelope.schema.json"),
|
||||
*manifest_assets,
|
||||
(downloads / "files.csv", "File hashes CSV", "text/csv", None),
|
||||
(downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)",
|
||||
"application/xml", None),
|
||||
@@ -1531,6 +1592,7 @@ def generate_data_exports(
|
||||
"generated_at": generated_at,
|
||||
**stamp,
|
||||
"kind": "catalog",
|
||||
"license": DATA_LICENSE,
|
||||
"count": len(published),
|
||||
"items": published,
|
||||
}
|
||||
@@ -1538,10 +1600,176 @@ def generate_data_exports(
|
||||
str(api / "catalog.json"),
|
||||
json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
)
|
||||
# api/v1/ itself is a folder GitHub Pages answers with the 404 page: the
|
||||
# entry point is a document of its own, naming where to start.
|
||||
entry_points = [
|
||||
{"title": "Catalog", "url": "catalog.json",
|
||||
"description": "Every published file with its size, SHA256 and schema."},
|
||||
{"title": "System ids", "url": "systems.json",
|
||||
"description": "Each system id with its name and the section holding its files."},
|
||||
{"title": "Core names", "url": "cores.json",
|
||||
"description": "Each core name and alias with the profile it designates."},
|
||||
{"title": "Schemas", "url": "schemas/site-api-envelope.schema.json",
|
||||
"description": "The envelope every JSON document of api/v1 shares."},
|
||||
]
|
||||
write_if_changed(str(api / "index.json"), json.dumps({
|
||||
"schema_version": 1, "generated_at": generated_at, **stamp,
|
||||
"kind": "index", "license": DATA_LICENSE,
|
||||
"count": len(entry_points), "items": entry_points,
|
||||
}, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||||
return catalog_items
|
||||
|
||||
|
||||
def _publish_manifests(api: Path) -> list[tuple]:
|
||||
"""The install manifests and the release record, beside the API.
|
||||
|
||||
The platform's list says what a setup checks. The manifest says what its
|
||||
pack installs: Batocera's list holds 353 files, its pack 14,508. The
|
||||
only copy was install/<platform>.json on the repository's main branch,
|
||||
which no page named. Copied byte for byte: the SHA256 the catalog gives
|
||||
is the repository file's.
|
||||
"""
|
||||
folder = api / "manifests"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
assets: list[tuple] = []
|
||||
for source in sorted((_REPO_ROOT / "install").glob("*.json")):
|
||||
target = folder / source.name
|
||||
shutil.copy2(source, target)
|
||||
_record(target)
|
||||
assets.append((target, f"Install manifest: {source.stem}", "application/json",
|
||||
"schemas/install-manifest.schema.json"))
|
||||
release = _REPO_ROOT / "release.json"
|
||||
if release.is_file():
|
||||
shutil.copy2(release, api / "release.json")
|
||||
_record(api / "release.json")
|
||||
assets.append((api / "release.json", "Latest release: packs, sizes and assets",
|
||||
"application/json", "schemas/release.schema.json"))
|
||||
return assets
|
||||
|
||||
|
||||
def _site_path(markdown_target: str) -> str:
|
||||
"""A page link as generators write it (systems/sony.md#psx) as a
|
||||
site-absolute path (/retrobios/systems/sony/#psx), which resolves the same
|
||||
from any document that holds it."""
|
||||
page, _, anchor = markdown_target.partition("#")
|
||||
base = urllib.parse.urlsplit(SITE_URL).path
|
||||
page = page.removesuffix(".md")
|
||||
page = "" if page == "index" else page.removesuffix("/index") + "/"
|
||||
return f"{base}{page}" + (f"#{anchor}" if anchor else "")
|
||||
|
||||
|
||||
def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]:
|
||||
"""Each system id the platforms and profiles write, reachable by that id.
|
||||
|
||||
api/v1/systems/ is keyed by manufacturer and its consoles by display
|
||||
name: a frontend keying by system id (sony-playstation) had no way in.
|
||||
The url is the section system_section gives, the one every page links.
|
||||
"""
|
||||
from siterender import system_ids, system_section
|
||||
|
||||
sha1_by_path = _sha1_by_path(db)
|
||||
files = db.get("files") or {}
|
||||
by_md5 = (db.get("indexes") or {}).get("by_md5") or {}
|
||||
held: dict[str, set[str]] = {}
|
||||
for coverage in coverages.values():
|
||||
for detail in coverage.get("details") or []:
|
||||
path = str(detail.get("path") or "")
|
||||
sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None
|
||||
if sha1:
|
||||
held.setdefault(str(detail.get("system", "")), set()).add(sha1)
|
||||
for profile in profiles.values():
|
||||
if profile.get("type") in ("alias", "test"):
|
||||
continue
|
||||
own = [str(s) for s in profile.get("systems") or []]
|
||||
for entry in profile.get("files") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
ids = ([str(entry["system"])] if entry.get("system")
|
||||
else [str(s) for s in entry.get("systems") or []] or
|
||||
(own if len(own) == 1 else []))
|
||||
found = {v for v in _hash_values(entry.get("sha1")) if v in files}
|
||||
for md5 in _hash_values(entry.get("md5")):
|
||||
match = by_md5.get(md5)
|
||||
found.update([match] if isinstance(match, str) else match or [])
|
||||
for sys_id in ids:
|
||||
held.setdefault(sys_id, set()).update(found)
|
||||
manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)}
|
||||
items = []
|
||||
for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])):
|
||||
section = system_section(sys_id)
|
||||
try:
|
||||
name = system_name(sys_id)
|
||||
except KeyError:
|
||||
name = sys_id
|
||||
items.append({
|
||||
"id": sys_id,
|
||||
"name": name,
|
||||
"manufacturer": manufacturers.get(section[0], "") if section else "",
|
||||
"url": _site_path(console_href(*section)) if section else None,
|
||||
"sha1": sorted(held.get(sys_id, set()) & files.keys()),
|
||||
})
|
||||
return items
|
||||
|
||||
|
||||
def core_index_items(profiles: dict) -> list[dict]:
|
||||
"""Every name a core goes by, with the profile it designates.
|
||||
|
||||
A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only
|
||||
beetle_psx.json: the reverse index resolve_platform_cores reads, aliases
|
||||
included, said once. A name two profiles claim (pcsx2) lists both, the
|
||||
one a platform list designates first.
|
||||
"""
|
||||
from common import preferred_profile, upstream_profile_index
|
||||
|
||||
published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")}
|
||||
index = upstream_profile_index(profiles)
|
||||
for name, profile in profiles.items():
|
||||
target = str(profile.get("alias_of") or "")
|
||||
if profile.get("type") == "alias" and target in published:
|
||||
index.setdefault(name, set()).add(target)
|
||||
items = []
|
||||
for name in sorted(index):
|
||||
claimants = sorted(index[name] & published)
|
||||
preferred = preferred_profile({name: set(claimants)}, name)
|
||||
for profile in sorted(claimants, key=lambda p: (p != preferred, p)):
|
||||
items.append({"name": name, "profile": profile, "preferred": profile == preferred})
|
||||
return items
|
||||
|
||||
|
||||
def _is_manifest(item: dict) -> bool:
|
||||
return item["url"].startswith("api/v1/manifests/")
|
||||
|
||||
|
||||
def _schema_field_rows(schema: dict, keys: tuple[str, ...]) -> list[str]:
|
||||
"""One table row per field, its description read from the schema, so the
|
||||
page and the published contract say the same thing."""
|
||||
props = schema.get("properties") or {}
|
||||
return [
|
||||
f"| `{key}` | {_escape_tags(str(props[key]['description']))} |"
|
||||
for key in keys if (props.get(key) or {}).get("description")
|
||||
]
|
||||
|
||||
|
||||
# The manifest fields an integrator reads first, in the order they matter.
|
||||
MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files",
|
||||
"omitted_files", "pack_files", "core_names", "system_cores")
|
||||
MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores",
|
||||
"systems", "category", "alternatives")
|
||||
|
||||
|
||||
def _short_sha256(item: dict, catalog: str) -> str:
|
||||
"""A digest cell that fits its column: 12 characters linking the catalog
|
||||
that holds it, the whole value in the title. 64 characters pushed the
|
||||
table to 932 px in a 720 px column."""
|
||||
value = str(item["sha256"])
|
||||
return f'<a href="{catalog}"><code title="{value}">{value[:12]}</code></a>'
|
||||
|
||||
|
||||
def generate_data_page(exports: list[dict]) -> str:
|
||||
manifest_schema = json.loads(
|
||||
(_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8"))
|
||||
snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
|
||||
manifests = [item for item in exports if _is_manifest(item)]
|
||||
lines = [
|
||||
"# Data and API",
|
||||
"",
|
||||
@@ -1554,20 +1782,21 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
markup.button("Collection DAT", "downloads/retrobios.dat"),
|
||||
]),
|
||||
"",
|
||||
"The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
|
||||
"per `bios/<Manufacturer>/<Console>/` folder of the collection, named "
|
||||
"`Manufacturer\\Console` so the tools look in that subfolder, each file a "
|
||||
"rom named by its path below that folder. It audits a copy of the "
|
||||
"collection's `bios/` tree, or rebuilds that layout from your own dumps. A "
|
||||
"platform pack has its own layout and is checked with `install.py --check`.",
|
||||
f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), "
|
||||
"which cover the scripts, tooling, schemas, documentation and metadata. "
|
||||
f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). "
|
||||
"`catalog.json` and the SQLite `metadata` table carry the same `license`.",
|
||||
"",
|
||||
"## Stable JSON endpoints",
|
||||
"",
|
||||
"Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, "
|
||||
"the two id indexes and the envelope schema.",
|
||||
"",
|
||||
"| Dataset | Endpoint | Schema | Size |",
|
||||
"|---------|----------|--------|-----:|",
|
||||
]
|
||||
for item in exports:
|
||||
if item["media_type"] != "application/json":
|
||||
if item["media_type"] != "application/json" or _is_manifest(item):
|
||||
continue
|
||||
schema = (
|
||||
f"[JSON Schema](api/v1/{item['schema']})"
|
||||
@@ -1579,25 +1808,96 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
)
|
||||
lines.extend([
|
||||
"",
|
||||
"Every JSON document carries `schema_version`, and `commit`, the revision "
|
||||
"Every envelope carries `schema_version`, `kind` and `commit`, the revision "
|
||||
"of this repository the data was read at. Breaking changes use a new "
|
||||
"URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.",
|
||||
"URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. "
|
||||
"Each schema's `$id` is the address it is served at.",
|
||||
"",
|
||||
"One emulator profile without the whole list: "
|
||||
"`api/v1/emulators/<id>.json`, the envelope of `emulators.json` holding "
|
||||
"that one item, its `profile` valid against "
|
||||
"[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each "
|
||||
"emulator page links its own. `api/v1/platforms/<id>.json` holds one "
|
||||
"platform of `platforms.json`, and `api/v1/systems/<slug>.json` the files "
|
||||
"of one system page by console. Each platform and system page links its own.",
|
||||
"platform of `platforms.json`, its `contract` valid against "
|
||||
"[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its "
|
||||
"`registry` the install and scraper settings beside the list. "
|
||||
"`api/v1/systems/<slug>.json` holds the files of one system page by console. "
|
||||
"Each platform and system page links its own.",
|
||||
"",
|
||||
"By id: [`systems.json`](api/v1/systems.json) gives each system id a platform "
|
||||
"or a profile writes (`sony-playstation`) with its name, its manufacturer, the "
|
||||
"`url` of the section holding its files and their SHA1. "
|
||||
"[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a "
|
||||
"platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), "
|
||||
"whose document is `api/v1/emulators/<profile>.json`.",
|
||||
"",
|
||||
"One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
|
||||
"is the first two characters of the hash in lowercase, holds every SHA1, "
|
||||
"MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
|
||||
"(the collection, a data directory, a platform's list or an emulator "
|
||||
"profile), the file `name` and the `url` of the page that shows it. "
|
||||
"profile), the file `name`, the `url` of the page that shows it and, "
|
||||
"when one is known, the `system_name` the site gives its system. "
|
||||
"[Find a file](files/index.md) reads it.",
|
||||
"",
|
||||
"Two kinds of `url`:",
|
||||
"",
|
||||
"- In `catalog.json` and `index.json`, a `url` is relative to the document "
|
||||
"holding it, as RFC 3986 resolves it.",
|
||||
"- In the lookup shards, `file-locations.json` and `systems.json`, a `url` "
|
||||
f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix "
|
||||
f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.",
|
||||
"",
|
||||
"## Platform manifests",
|
||||
"",
|
||||
"A platform's list is what the setup checks. Its manifest is what the pack "
|
||||
"installs: the list's files plus every file the setup's cores load. "
|
||||
"`api/v1/manifests/<platform>.json` is the file `install.py` reads, byte "
|
||||
"for byte, valid against "
|
||||
"[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). "
|
||||
"[`api/v1/release.json`](api/v1/release.json) records the packs of the latest "
|
||||
"release and the files each one holds.",
|
||||
"",
|
||||
"Manifests: " + ", ".join(
|
||||
f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})"
|
||||
for item in manifests) + "." if manifests else "",
|
||||
"",
|
||||
"| Field | Meaning |",
|
||||
"|-------|---------|",
|
||||
*_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS),
|
||||
"",
|
||||
"Each entry of `files`:",
|
||||
"",
|
||||
"| Field | Meaning |",
|
||||
"|-------|---------|",
|
||||
*_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS),
|
||||
"",
|
||||
"## Check a folder for one platform",
|
||||
"",
|
||||
"The job most tools come for: is this BIOS folder what the platform checks? "
|
||||
"Read `api/v1/platforms/<id>.json` and apply the platform's own rules:",
|
||||
"",
|
||||
"- `verification_mode: existence`: a file at its `destination` passes.",
|
||||
"- An entry with no hash in the mode's field is checked on presence too.",
|
||||
"- `zipped_file` set: the md5 is that of this member inside the ZIP, never "
|
||||
"the ZIP's. Member names compare casefolded. A ZIP declared once per member "
|
||||
"appears once per member: Batocera lists 692 entries for 353 destinations.",
|
||||
"- An md5 shorter than 32 characters (Batocera) compares by its prefix. "
|
||||
"Recalbox writes several accepted md5 in one field, comma separated.",
|
||||
"- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members "
|
||||
"read in name order, never the archive's.",
|
||||
"- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. "
|
||||
"The script below does not apply it.",
|
||||
"",
|
||||
"```python",
|
||||
snippet.rstrip("\n"),
|
||||
"```",
|
||||
"",
|
||||
"The script is [`scripts/check_platform_folder.py`]"
|
||||
f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it "
|
||||
"against the verifier on a sample folder, so the two give the same answer. "
|
||||
"`install.py --check` answers another question: whether the folder holds "
|
||||
"what the pack installs, the manifest above.",
|
||||
"",
|
||||
"## Bulk downloads",
|
||||
"",
|
||||
"| Export | Format | Size | SHA256 |",
|
||||
@@ -1608,9 +1908,19 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
continue
|
||||
lines.append(
|
||||
f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | "
|
||||
f"{_fmt_size(item['bytes'])} | `{item['sha256']}` |"
|
||||
f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |"
|
||||
)
|
||||
lines.extend([
|
||||
"",
|
||||
"Each SHA256 is cut to 12 characters. Point at it for the whole value, or "
|
||||
"read it in [`catalog.json`](api/v1/catalog.json).",
|
||||
"",
|
||||
"The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
|
||||
"per `bios/<Manufacturer>/<Console>/` folder of the collection, named "
|
||||
"`Manufacturer\\Console` so the tools look in that subfolder, each file a "
|
||||
"rom named by its path below that folder. It audits a copy of the "
|
||||
"collection's `bios/` tree, or rebuilds that layout from your own dumps. A "
|
||||
"platform pack has its own layout and is checked with `install.py --check`.",
|
||||
"",
|
||||
"The SQLite file contains indexed tables for content hashes, provenance "
|
||||
"catalog matches, platform declarations, emulator profiles and current "
|
||||
@@ -1644,6 +1954,25 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
"`provenance-wanted.csv` is the Wanted list of [Dump provenance]"
|
||||
"(provenance.md): each dump a catalog describes and the collection lacks.",
|
||||
"",
|
||||
"`platform-files.csv` and the `platform_files` table also carry the entry "
|
||||
"fields a check needs:",
|
||||
"",
|
||||
"- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.",
|
||||
"- `zip_digest` is `composite` where the md5 of a ZIP covers its members.",
|
||||
"- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.",
|
||||
"- `curated`, `shared_group` and `mirror_of` mark an entry the platform's "
|
||||
"own file does not carry.",
|
||||
"",
|
||||
"## Hosting and limits",
|
||||
"",
|
||||
"- Static files on GitHub Pages, read with a plain GET. No key, no account.",
|
||||
"- Any origin may read them (`Access-Control-Allow-Origin: *`), so a "
|
||||
"browser tool can call them directly.",
|
||||
"- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). "
|
||||
"Send the `ETag` back in `If-None-Match` to skip an unchanged download.",
|
||||
"- The base is the project's address, "
|
||||
f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.",
|
||||
"",
|
||||
"## Semantics and limits",
|
||||
"",
|
||||
"Treat four questions separately: content identity (hashes), presence in "
|
||||
@@ -2404,19 +2733,30 @@ def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]:
|
||||
return runs
|
||||
|
||||
|
||||
def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]:
|
||||
"""The `parent:` front matter of a part page: the page it belongs to,
|
||||
by label and site path, which the breadcrumb and sidebar read."""
|
||||
if parent is None:
|
||||
return []
|
||||
label, link = parent
|
||||
return ["parent:", f" label: {json.dumps(label)}", f" link: {json.dumps(link)}"]
|
||||
|
||||
|
||||
def _list_pages(
|
||||
stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]],
|
||||
moved: dict[str, str], budget: int = LIST_PART_BUDGET,
|
||||
parent: tuple[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Pages of their own for a long list: ``head`` (a table header, say)
|
||||
opens each part, ``blocks`` are cut between parts. Returns where the list
|
||||
went, as the page it left says it."""
|
||||
went, as the page it left says it. ``parent`` (label, site path) is the
|
||||
page the parts belong to, for the breadcrumb."""
|
||||
runs = _blocks_in_runs(blocks, budget)
|
||||
stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)]
|
||||
for n, (here, run) in enumerate(zip(stems, runs), 1):
|
||||
part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
|
||||
moved[here] = "\n".join([
|
||||
"---", "level: reference", "---", "",
|
||||
"---", "level: reference", *_parent_front_matter(parent), "---", "",
|
||||
f"# {title}{part}", "", intro, "",
|
||||
*([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}"
|
||||
for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []),
|
||||
@@ -2443,6 +2783,11 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str:
|
||||
writes. A system with no section of its own is named, never sent to the
|
||||
page a storage folder chose.
|
||||
"""
|
||||
return f"{_system_named_link(sys_id, prefix)} <small>{_escape_tags(sys_id)}</small>"
|
||||
|
||||
|
||||
def _system_named_link(sys_id: str, prefix: str = "") -> str:
|
||||
"""A system's name, linked to the section system_section gives."""
|
||||
from siterender import system_section
|
||||
|
||||
name = _escape_tags(system_name(sys_id))
|
||||
@@ -2450,7 +2795,17 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str:
|
||||
if section is not None:
|
||||
slug, console = section
|
||||
name = f"[{name}]({console_href(slug, console, prefix)})"
|
||||
return f"{name} <small>{_escape_tags(sys_id)}</small>"
|
||||
return name
|
||||
|
||||
|
||||
def _emulator_named_link(key: str, prefix: str = "") -> str:
|
||||
"""An emulator by the name its page carries, linked to that page. A
|
||||
profile with no page (an alias, a test) is named only."""
|
||||
profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {})
|
||||
label = _escape_tags(str(profile.get("emulator") or key))
|
||||
if profile.get("type") in ("alias", "test"):
|
||||
return label
|
||||
return f"[{label}]({prefix}emulators/{key}.md)"
|
||||
|
||||
|
||||
def _served_line(f: dict) -> str:
|
||||
@@ -2532,7 +2887,7 @@ def _render_platform_systems(
|
||||
anchor = sys_id.replace(" ", "-")
|
||||
body = [f"{ok_count}/{total} files verified"]
|
||||
if sys_emus:
|
||||
emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus))
|
||||
emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus))
|
||||
body[-1] += "\\"
|
||||
body.append(f"Emulators: {emu_links}")
|
||||
body.append("")
|
||||
@@ -2659,8 +3014,12 @@ def _render_platform_systems(
|
||||
for n in range(1, len(runs) + 1)]
|
||||
for n, (stem, run) in enumerate(zip(stems, runs), 1):
|
||||
part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
|
||||
# A console query landed here with no way to the console's
|
||||
# own page: the list says whose it is first, linked.
|
||||
moved[stem] = "\n".join([
|
||||
"---", *_parent_front_matter((display, f"/platforms/{key}/")), "---", "",
|
||||
f"# {display}: {system_name(sys_id)}{part}", "",
|
||||
f"{display}'s list for {_system_named_link(sys_id, '../')}. "
|
||||
f"Part of the [{display} page]({key}.md#{anchor}), whose other systems "
|
||||
f"are listed there. Each file below is as {display} declares it, with "
|
||||
"its status in the pack.",
|
||||
@@ -3254,10 +3613,12 @@ def generate_platform_page(
|
||||
f'{non_ok} issue{"s" if non_ok > 1 else ""}</span>'
|
||||
)
|
||||
sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id))
|
||||
emu_str = ", ".join(sys_emus[:3])
|
||||
if len(sys_emus) > 3:
|
||||
emu_str += f" +{len(sys_emus) - 3}"
|
||||
anchor = sys_id.replace(" ", "-")
|
||||
# Named and linked as the detail blocks below do: the column held
|
||||
# profile ids, none of 1,463 linked.
|
||||
emu_str = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)[:3])
|
||||
if len(sys_emus) > 3:
|
||||
emu_str += f" [+{len(sys_emus) - 3}](#{anchor})"
|
||||
# The name a reader filters on, the id the list writes beside it.
|
||||
lines.append(
|
||||
f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) <small>{sys_id}</small> "
|
||||
@@ -3390,7 +3751,9 @@ def file_location_items(
|
||||
A card lives on its manufacturer's page, and a file refiled between
|
||||
manufacturers (mt32_control.rom, Commodore to Roland) moved its link
|
||||
with it. files/?sha1= reads this list and goes to the card wherever it
|
||||
is now: the address cited depends on the content only.
|
||||
is now: the address cited depends on the content only. The url is
|
||||
site-absolute (/retrobios/systems/...): written from the site root, it
|
||||
resolved against the document holding it to a page that does not exist.
|
||||
"""
|
||||
items = []
|
||||
for sha1, entry in sorted(db.get("files", {}).items()):
|
||||
@@ -3398,7 +3761,7 @@ def file_location_items(
|
||||
if len(parts) < 3 or parts[0] != "bios":
|
||||
continue
|
||||
page = card_page(entry.get("path", ""), sha1)
|
||||
items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"})
|
||||
items.append({"sha1": sha1, "url": _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")})
|
||||
# A data directory's files are listed on the page of an emulator reading
|
||||
# the tree: 2,534 hashes the packs ship were found nowhere on the site.
|
||||
owners = _data_member_owners(profiles or {})
|
||||
@@ -3414,13 +3777,38 @@ def file_location_items(
|
||||
if sha1 and sha1 not in known:
|
||||
known.add(sha1)
|
||||
page = page_of.get(member, owners[ref])
|
||||
items.append({"sha1": sha1, "url": f"emulators/{page}/#sha1-{sha1[:12]}"})
|
||||
items.append({"sha1": sha1,
|
||||
"url": _site_path(f"emulators/{page}.md#sha1-{sha1[:12]}")})
|
||||
return sorted(items, key=lambda item: item["sha1"])
|
||||
|
||||
|
||||
LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files
|
||||
|
||||
|
||||
def _lookup_system_name(sys_id: str) -> str | None:
|
||||
"""The system's name as every page prints it, None for an id the site
|
||||
cannot name. The Find a file page reads it, and never builds one."""
|
||||
try:
|
||||
return system_name(sys_id)
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
|
||||
def _console_system_names(profiles: dict, coverages: dict) -> dict[tuple[str, str], str]:
|
||||
"""(manufacturer slug, console folder) -> the name of the system whose
|
||||
own section that folder is: a collection file is named after the console
|
||||
it is filed under. A folder no id owns, or that two names own, is left out.
|
||||
"""
|
||||
from siterender import _own_section, system_ids
|
||||
|
||||
found: dict[tuple[str, str], set[str]] = {}
|
||||
for sys_id in system_ids(profiles, [c["config"] for c in coverages.values()]):
|
||||
own = _own_section(sys_id)
|
||||
if own is not None:
|
||||
found.setdefault((_slug(own[0]), own[1]), set()).add(system_name(sys_id))
|
||||
return {key: next(iter(names)) for key, names in found.items() if len(names) == 1}
|
||||
|
||||
|
||||
def file_lookup_records(
|
||||
db: dict,
|
||||
coverages: dict,
|
||||
@@ -3436,6 +3824,7 @@ def file_lookup_records(
|
||||
it made every visitor download: the lookup reads one shard instead.
|
||||
"""
|
||||
files = db.get("files", {})
|
||||
named = _console_system_names(profiles, coverages)
|
||||
member_names = {
|
||||
str(info.get("sha1", "")): member.rsplit("/", 1)[-1]
|
||||
for tree in (members or {}).values() for member, info in sorted(tree.items())
|
||||
@@ -3450,28 +3839,37 @@ def file_lookup_records(
|
||||
"name": member_names.get(sha1, ""), "sha1": sha1,
|
||||
"url": item["url"]})
|
||||
continue
|
||||
parts = str(entry.get("path", "")).split("/")
|
||||
held_by = named.get((_slug(parts[1]), parts[2])) if len(parts) > 3 else None
|
||||
for algorithm in HASH_ALGORITHMS:
|
||||
for value in _hash_values(entry.get(algorithm)):
|
||||
records.append({
|
||||
"value": value, "algorithm": algorithm, "layer": "collection",
|
||||
"name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"],
|
||||
**({"system_name": held_by} if held_by else {}),
|
||||
})
|
||||
for key, coverage in sorted(coverages.items()):
|
||||
for system_id, system in sorted((coverage["config"].get("systems") or {}).items()):
|
||||
url = f"platforms/{key}/#{system_id.replace(' ', '-')}"
|
||||
url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}")
|
||||
system_label = _lookup_system_name(system_id)
|
||||
for entry in system.get("files") or []:
|
||||
for algorithm in HASH_ALGORITHMS:
|
||||
for value in _hash_values(entry.get(algorithm)):
|
||||
records.append({
|
||||
"value": value, "algorithm": algorithm, "layer": "platform",
|
||||
"owner": key, "name": str(entry.get("name", "")), "url": url,
|
||||
**({"system_name": system_label} if system_label else {}),
|
||||
})
|
||||
for key, profile in sorted(profiles.items()):
|
||||
if profile.get("type") in ("alias", "test"):
|
||||
continue
|
||||
entries = profile.get("files") or []
|
||||
own_systems = [str(s) for s in profile.get("systems") or []]
|
||||
for entry, anchor in zip(entries, emulator_file_anchors(entries)):
|
||||
url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}"
|
||||
url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}")
|
||||
sys_id = str(entry.get("system") or "") or (
|
||||
own_systems[0] if len(own_systems) == 1 else "")
|
||||
entry_label = _lookup_system_name(sys_id) if sys_id else None
|
||||
members = [(str(entry.get("name", "")), entry)] + [
|
||||
(str(member["name"]), member) for member in entry.get("contents") or []
|
||||
if isinstance(member, dict) and member.get("name")
|
||||
@@ -3482,6 +3880,7 @@ def file_lookup_records(
|
||||
records.append({
|
||||
"value": value, "algorithm": algorithm, "layer": "emulator",
|
||||
"owner": key, "name": name, "url": url,
|
||||
**({"system_name": entry_label} if entry_label else {}),
|
||||
})
|
||||
unique = {json.dumps(r, sort_keys=True): r for r in records}
|
||||
return sorted(unique.values(), key=lambda r: (
|
||||
@@ -3883,7 +4282,7 @@ def generate_provenance_page(
|
||||
f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}",
|
||||
f"Dumps {label} catalogues and the collection lacks, by DAT. Part of "
|
||||
f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).",
|
||||
[], blocks, moved,
|
||||
[], blocks, moved, parent=("Dump provenance", "/provenance/"),
|
||||
)
|
||||
lines.extend([
|
||||
f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: "
|
||||
@@ -6727,12 +7126,27 @@ def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]:
|
||||
return {}
|
||||
|
||||
|
||||
def short_hash(value: str) -> str:
|
||||
"""A hash in a table cell: its first 12 characters, the whole value in
|
||||
the title. Full MD5 and SHA1 values pushed the gaps tables past the
|
||||
column on every screen."""
|
||||
value = str(value)
|
||||
if len(value) <= 12:
|
||||
return _code(value)
|
||||
return f'<code title="{html.escape(value)}">{html.escape(value[:12])}</code>'
|
||||
|
||||
|
||||
def _hash_cell(hashes: dict[str, list[str]]) -> str:
|
||||
return "<br>".join(
|
||||
f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items()
|
||||
f"{algo} " + ", ".join(short_hash(v) for v in values) for algo, values in hashes.items()
|
||||
)
|
||||
|
||||
|
||||
def _reason_cell(text: str) -> str:
|
||||
"""A reason in a gaps table, in the class that keeps its column readable."""
|
||||
return f'<span class="rb-reason">{text}</span>' if text else "-"
|
||||
|
||||
|
||||
def _copy_cell(path: str, algos: list[str], db: dict | None) -> str:
|
||||
"""The file a path holds, with its hashes in the algorithms declared."""
|
||||
if not path:
|
||||
@@ -6843,7 +7257,7 @@ def _render_problem_files(
|
||||
plats = ", ".join(sorted(set(f["platforms"])))
|
||||
declared = f["declared"]
|
||||
lines.append(
|
||||
f"| {_code(f['name'])} | {plats} | {f['reason']} "
|
||||
f"| {_code(f['name'])} | {plats} | {_reason_cell(f['reason'])} "
|
||||
f"| {_hash_cell(declared) or '-'} "
|
||||
f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |"
|
||||
)
|
||||
@@ -6865,7 +7279,7 @@ def _render_problem_files(
|
||||
plats = ", ".join(sorted(set(f["platforms"])))
|
||||
held = _held_accepted(f["discrepancy"], db)
|
||||
cell = f"[{_code(held)}]({_fetch_url(held)})" if held else "-"
|
||||
lines.append(f"| `{fname}` | {plats} | {f['discrepancy']} | {cell} |")
|
||||
lines.append(f"| `{fname}` | {plats} | {_reason_cell(f['discrepancy'])} | {cell} |")
|
||||
lines.append("")
|
||||
|
||||
if settled:
|
||||
@@ -7366,7 +7780,7 @@ def generate_gap_analysis(
|
||||
)
|
||||
req = "yes" if m["required"] else "no"
|
||||
algorithm, _, value = m["identity"].partition(":")
|
||||
expected = f"{algorithm} {_code(value)}" if value else "-"
|
||||
expected = f"{algorithm} {short_hash(value)}" if value else "-"
|
||||
destination = _code(m["path"]) if m["path"] != m["name"] else "-"
|
||||
emulators = ", ".join(
|
||||
f"[{_escape_tags(display)}](emulators/{key}.md)"
|
||||
@@ -7387,6 +7801,7 @@ def generate_gap_analysis(
|
||||
"the collection does not hold. Part of "
|
||||
"[What the packs do not hold](gaps.md#files-to-source).",
|
||||
head, [[row] for row in rows], moved,
|
||||
parent=("What the packs do not hold", "/gaps/"),
|
||||
)
|
||||
lines.extend([f"Listed on {where}.", ""])
|
||||
if bundled_rows:
|
||||
@@ -7423,7 +7838,7 @@ def generate_gap_analysis(
|
||||
])
|
||||
for u in sorted(all_unsourceable, key=lambda x: x["name"]):
|
||||
lines.append(
|
||||
f"| `{u['name']}` | {u['emulator']} | {u['reason']} "
|
||||
f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} "
|
||||
f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |"
|
||||
)
|
||||
lines.append("")
|
||||
@@ -7588,7 +8003,8 @@ def _render_xref_by_platform(
|
||||
)
|
||||
if moved is not None:
|
||||
moved[stem] = "\n".join([
|
||||
"---", "level: reference", "---", "",
|
||||
"---", "level: reference",
|
||||
*_parent_front_matter(("Cores by platform", "/cross-reference/")), "---", "",
|
||||
f"# Cores on {display}",
|
||||
"",
|
||||
f"{_count(count, 'core')}{_core_scope_note(config, unique)} that "
|
||||
@@ -8529,7 +8945,7 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
write_single_item(
|
||||
public / "api" / "v1" / "systems" / f"{slug}.json", "systems",
|
||||
system_export_item(slug, mfr, consoles),
|
||||
str(db.get("generated_at") or _timestamp()),
|
||||
str(db.get("generated_at") or _timestamp()), _source_commit(),
|
||||
)
|
||||
clash = split_system_stems() & {_slug(m) for m in manufacturers}
|
||||
if clash:
|
||||
@@ -8612,18 +9028,11 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
# The envelope and item of emulators.json, one item long: the
|
||||
# profile validates against the published schema, which an "id"
|
||||
# beside its fields broke on all 530, and schema_version is there.
|
||||
write_if_changed(
|
||||
str(public / "api" / "v1" / "emulators" / f"{name}.json"),
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"generated_at": str(db.get("generated_at") or _timestamp()),
|
||||
"kind": "emulators",
|
||||
"count": 1,
|
||||
"items": [{"id": name, "profile": profile}],
|
||||
},
|
||||
indent=2, sort_keys=True, ensure_ascii=False, default=str,
|
||||
) + "\n",
|
||||
# The commit too, as every other envelope: 625 files lacked it.
|
||||
write_single_item(
|
||||
public / "api" / "v1" / "emulators" / f"{name}.json", "emulators",
|
||||
{"id": name, "profile": profile},
|
||||
str(db.get("generated_at") or _timestamp()), _source_commit(),
|
||||
)
|
||||
pages = emulator_pages(
|
||||
name, profile, db, consumers, suppl_names, data_registry,
|
||||
|
||||
+50
-2
@@ -5,9 +5,11 @@ was written against, so a permalink is pinned, never guessed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import re
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
|
||||
import upstream
|
||||
@@ -150,6 +152,45 @@ def _source_permalink(repo: upstream.Repo, pin: str, path: str,
|
||||
url += f"-L{end}"
|
||||
return url
|
||||
|
||||
# The tree listings profile_sync caches. Read offline only: a page build
|
||||
# never calls a forge, and a listing it does not find leaves the link as the
|
||||
# profile's declarations alone decide it.
|
||||
TREE_CACHE = Path(__file__).resolve().parent.parent / upstream.CACHE_DIR
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str] | None:
|
||||
"""Every file of a repository at a revision, None when not cached or
|
||||
truncated (a missing path would then prove nothing)."""
|
||||
if not repo.api_base:
|
||||
return None # a browse-only forge: no listing to read
|
||||
paths, truncated = upstream.list_tree(repo, pin, str(TREE_CACHE), offline=True)
|
||||
return frozenset(paths) if paths and not truncated else None
|
||||
|
||||
|
||||
def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool | None:
|
||||
"""Whether the repository holds ``path`` at ``pin``: None when its tree
|
||||
is not cached. trident cites src/core/loader/ncch.cpp, a Panda3DS file,
|
||||
and the page linked it into 3dsTrident, where it is a 404."""
|
||||
paths = _tree_paths(repo, pin)
|
||||
return None if paths is None else path in paths
|
||||
|
||||
|
||||
def _holder(
|
||||
candidates: list[tuple[upstream.Repo, str]], path: str, default: tuple | None,
|
||||
) -> tuple[upstream.Repo, str] | None:
|
||||
"""The pinned repository that holds ``path``, source before upstream as
|
||||
profile_sync.select_views orders them. Without every tree cached, the
|
||||
declarations decide (``default``). A path no cached tree holds links
|
||||
nowhere: no link beats a link to a file that is not there."""
|
||||
verdicts = [tree_holds(repo, pin, path) for repo, pin in candidates]
|
||||
if not candidates or None in verdicts:
|
||||
if default is not None and tree_holds(*default, path) is False:
|
||||
return None
|
||||
return default
|
||||
return next((cand for cand, held in zip(candidates, verdicts) if held), None)
|
||||
|
||||
|
||||
def _commit_url(repo: upstream.Repo, pin: str) -> str:
|
||||
"""Forge-specific browser URL for one commit."""
|
||||
base = f"https://{repo.host}/{repo.owner}/{repo.name}"
|
||||
@@ -272,8 +313,13 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
|
||||
if current_role:
|
||||
# Only the repository the part names, or no link at all.
|
||||
selected = by_role.get(current_role)
|
||||
elif len(candidates) == 1:
|
||||
selected = candidates[0]
|
||||
if selected is not None and tree_holds(*selected, part.path) is False:
|
||||
selected = None
|
||||
else:
|
||||
selected = _holder(
|
||||
candidates, part.path,
|
||||
candidates[0] if len(candidates) == 1 else None,
|
||||
)
|
||||
|
||||
tag = (f" ({role_label(profile, current_role, part.path)})"
|
||||
if current_role and _two_repositories(profile) else "")
|
||||
@@ -330,6 +376,8 @@ def link_prose_citations(profile: dict, text: str) -> str:
|
||||
continue
|
||||
if any(lo <= path_span[0] < hi for lo, hi in guarded):
|
||||
continue
|
||||
if tree_holds(repo, pin, path) is False:
|
||||
continue
|
||||
for index, ((lo, hi), part) in enumerate(zip(spans, parts)):
|
||||
start = path_span[0] if index == 0 else lo
|
||||
url = _source_permalink(repo, pin, part.path, part.start, part.end)
|
||||
|
||||
@@ -685,14 +685,14 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase):
|
||||
records = generate_site.file_lookup_records(db, coverages, profiles, locations, members)
|
||||
found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records}
|
||||
self.assertEqual(found[("collection", "", "md5", md5)]["url"],
|
||||
"systems/sony/#sha1-" + "ab" * 6)
|
||||
"/retrobios/systems/sony/#sha1-" + "ab" * 6)
|
||||
self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found)
|
||||
self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin")
|
||||
self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"],
|
||||
"platforms/batocera/#sony-playstation")
|
||||
"/retrobios/platforms/batocera/#sony-playstation")
|
||||
self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found)
|
||||
self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"],
|
||||
"emulators/duckstation/#file-scph5501-bin-490f666e")
|
||||
"/retrobios/emulators/duckstation/#file-scph5501-bin-490f666e")
|
||||
self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom")
|
||||
self.assertNotIn(("emulator", "an-alias", "md5", md5), found)
|
||||
|
||||
@@ -724,7 +724,7 @@ class AFileHasAnAddressByContent(unittest.TestCase):
|
||||
}}
|
||||
self.assertEqual(
|
||||
generate_site.file_location_items(db),
|
||||
[{"sha1": "ab" * 20, "url": "systems/roland/#sha1-" + "ab" * 6}],
|
||||
[{"sha1": "ab" * 20, "url": "/retrobios/systems/roland/#sha1-" + "ab" * 6}],
|
||||
)
|
||||
self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page())
|
||||
script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts"
|
||||
@@ -865,7 +865,8 @@ class ADataDirectoryFileCanBeLookedUp(unittest.TestCase):
|
||||
members = data_directory_members(Path(tmp))
|
||||
items = file_location_items(
|
||||
{"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, members)
|
||||
self.assertEqual(items, [{"sha1": "c" * 40, "url": "emulators/dolphin/#sha1-cccccccccccc"}])
|
||||
self.assertEqual(items, [{"sha1": "c" * 40,
|
||||
"url": "/retrobios/emulators/dolphin/#sha1-cccccccccccc"}])
|
||||
|
||||
|
||||
class ACardSaysWhyAFileIsLeftOut(unittest.TestCase):
|
||||
|
||||
@@ -69,8 +69,9 @@ class AnUntestedRowNamesBothHashes(unittest.TestCase):
|
||||
db = {"files": {"f" * 40: {"path": path, "sha256": held}}}
|
||||
table = "\n".join(gs._render_problem_files([problem], db=db))
|
||||
row = next(line for line in table.splitlines() if "dsifirmware" in line)
|
||||
self.assertIn(f"sha256 `{declared}`", row)
|
||||
self.assertIn(f"sha256 `{held}`", row)
|
||||
# Cut to 12 characters in the cell, the whole value in its title.
|
||||
self.assertIn(f'sha256 <code title="{declared}">{declared[:12]}</code>', row)
|
||||
self.assertIn(f'sha256 <code title="{held}">{held[:12]}</code>', row)
|
||||
self.assertIn(f"`{path}`", row)
|
||||
|
||||
|
||||
|
||||
@@ -261,7 +261,7 @@ class TheRules(unittest.TestCase):
|
||||
{"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}},
|
||||
{"dolphin-sys": members})
|
||||
last = f"{generate_site.DATA_TREE_ROWS_PER_PAGE:040x}"
|
||||
self.assertIn({"sha1": last, "url": f"emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items)
|
||||
self.assertIn({"sha1": last, "url": f"/retrobios/emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items)
|
||||
|
||||
def test_no_float_rule_for_right_aligned_images(self):
|
||||
css = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8")
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
"""Reference pages and data (round 3, package 8).
|
||||
|
||||
The CSV and SQLite dropped zipped_file, so a check built from them judged
|
||||
106 good Batocera files wrong. The manifest a pack installs was nowhere on
|
||||
the site. Contracts failed their own schema, 716 documents lacked `commit`,
|
||||
no endpoint answered a system id or a buildbot core name, lookup URLs
|
||||
resolved to 404s, the data's licence went unstated, platform overviews
|
||||
named emulators as unlinked ids, gaps tables overflowed with full hashes
|
||||
and a source link went into a repository that does not hold the file.
|
||||
Tests that read the built site skip when it has not been generated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import check_platform_folder # noqa: E402
|
||||
import generate_site # noqa: E402
|
||||
import sitesources # noqa: E402
|
||||
|
||||
PUBLIC = REPO_ROOT / "website" / "public"
|
||||
API = PUBLIC / "api" / "v1"
|
||||
SITE = "https://abdess.github.io/retrobios/"
|
||||
TMP = REPO_ROOT / "tmp"
|
||||
|
||||
|
||||
def _built(path: Path) -> Path:
|
||||
if not path.exists():
|
||||
raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated")
|
||||
return path
|
||||
|
||||
|
||||
def _verify_status(entry: dict, path: Path, mode: str) -> str:
|
||||
"""What verify.py says of one entry, the platform's native logic."""
|
||||
from verify import verify_entry_existence, verify_entry_md5, verify_entry_sha1
|
||||
|
||||
local = str(path) if path.is_file() else None
|
||||
if mode == "existence":
|
||||
return verify_entry_existence(entry, local)["status"]
|
||||
if mode == "sha1":
|
||||
return verify_entry_sha1(entry, local)["status"]
|
||||
return verify_entry_md5(entry, local)["status"]
|
||||
|
||||
|
||||
def _as_snippet(status: str) -> str:
|
||||
return {"ok": "ok", "untested": "wrong", "missing": "missing"}[status]
|
||||
|
||||
|
||||
class TheExportsKeepWhatACheckNeeds(unittest.TestCase):
|
||||
"""8.1: zipped_file and the Recalbox flags reach the CSV and SQLite."""
|
||||
|
||||
COVERAGE = {"batocera": {
|
||||
"platform": "Batocera", "details": [],
|
||||
"config": {"platform": "Batocera", "verification_mode": "md5",
|
||||
"install": {"detect": []}, "logo": "x.svg", "_key": "batocera",
|
||||
"systems": {"bbc": {"files": [
|
||||
{"name": "bbcb.zip", "destination": "bbcb.zip",
|
||||
"md5": "521a197d", "zipped_file": "basic1.rom"},
|
||||
{"name": "n.rom", "destination": "n.rom", "md5": "ab",
|
||||
"hash_match_mandatory": True, "mandatory_declared": False,
|
||||
"shared_group": "group-x"},
|
||||
]}}},
|
||||
}}
|
||||
|
||||
def test_rows_carry_the_member_and_the_flags(self):
|
||||
_items, rows = generate_site._platform_export_rows(self.COVERAGE)
|
||||
zipped, flagged = rows
|
||||
self.assertEqual(zipped["zipped_file"], "basic1.rom")
|
||||
self.assertIs(flagged["hash_match_mandatory"], True)
|
||||
self.assertIs(flagged["mandatory_declared"], False)
|
||||
self.assertEqual(flagged["shared_group"], "group-x")
|
||||
self.assertEqual(flagged["zipped_file"], "")
|
||||
|
||||
def test_the_csv_header_and_the_table_name_them(self):
|
||||
source = Path(generate_site.__file__).read_text(encoding="utf-8")
|
||||
self.assertIn('"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,', source)
|
||||
for field in generate_site.ENTRY_EXPORT_FIELDS:
|
||||
self.assertIn(f":{field}", source)
|
||||
for field in ("zipped_file", "hash_match_mandatory", "mandatory_declared", "curated"):
|
||||
self.assertIn(field, generate_site.ENTRY_EXPORT_FIELDS)
|
||||
|
||||
def test_the_built_csv_has_the_column(self):
|
||||
text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8")
|
||||
rows = list(csv.DictReader(io.StringIO(text)))
|
||||
batocera = [row for row in rows if row["platform_id"] == "batocera"]
|
||||
self.assertGreater(sum(1 for row in batocera if row["zipped_file"]), 300)
|
||||
with sqlite3.connect(_built(PUBLIC / "downloads" / "retrobios.sqlite")) as db:
|
||||
count = db.execute("SELECT count(*) FROM platform_files "
|
||||
"WHERE platform_id = 'batocera' AND zipped_file != ''").fetchone()[0]
|
||||
license_row = db.execute("SELECT value FROM metadata WHERE key = 'license'").fetchone()
|
||||
self.assertGreater(count, 300)
|
||||
self.assertEqual(license_row[0], generate_site.DATA_LICENSE)
|
||||
|
||||
|
||||
class AFolderCheckAgreesWithTheVerifier(unittest.TestCase):
|
||||
"""8.1 and 8.3: the published example gives verify.py's verdicts."""
|
||||
|
||||
def _folder(self, tmp: Path) -> tuple[dict, Path]:
|
||||
member = b"basic rom bytes"
|
||||
root = tmp / "bios"
|
||||
root.mkdir()
|
||||
with zipfile.ZipFile(root / "bbcb.zip", "w") as archive:
|
||||
archive.writestr("BASIC1.ROM", member)
|
||||
(root / "plain.bin").write_bytes(b"plain")
|
||||
(root / "wrong.bin").write_bytes(b"not it")
|
||||
(root / "nohash.bin").write_bytes(b"x")
|
||||
md5 = lambda data: hashlib.md5(data).hexdigest() # noqa: E731
|
||||
contract = {"verification_mode": "md5", "systems": {"s": {"files": [
|
||||
{"name": "bbcb.zip", "destination": "bbcb.zip", "md5": md5(member),
|
||||
"zipped_file": "basic1.rom"},
|
||||
{"name": "bbcb.zip", "destination": "bbcb.zip", "md5": "0" * 32,
|
||||
"zipped_file": "basic1.rom"},
|
||||
{"name": "plain.bin", "destination": "plain.bin", "md5": md5(b"plain")[:29]},
|
||||
{"name": "wrong.bin", "destination": "wrong.bin", "md5": md5(b"plain")},
|
||||
{"name": "nohash.bin", "destination": "nohash.bin"},
|
||||
{"name": "gone.bin", "destination": "gone.bin", "md5": "1" * 32},
|
||||
]}}}
|
||||
return contract, root
|
||||
|
||||
def test_each_rule_on_a_sample_folder(self):
|
||||
with tempfile.TemporaryDirectory(dir=TMP) as tmp:
|
||||
contract, root = self._folder(Path(tmp))
|
||||
platforms = Path(tmp) / "platforms"
|
||||
platforms.mkdir()
|
||||
(platforms / "sample.json").write_text(
|
||||
json.dumps({"items": [{"contract": contract}]}), encoding="utf-8")
|
||||
found = check_platform_folder.check("sample", root, platforms.as_uri() + "/")
|
||||
expected = [
|
||||
(entry["destination"], _as_snippet(_verify_status(entry, root / entry["destination"], "md5")))
|
||||
for entry in contract["systems"]["s"]["files"]
|
||||
]
|
||||
self.assertEqual(found, expected)
|
||||
self.assertEqual([v for _d, v in found], ["ok", "wrong", "ok", "wrong", "ok", "missing"])
|
||||
|
||||
def test_batocera_from_the_csv_matches_the_verifier(self):
|
||||
"""A check driven by platform-files.csv, with zipped_file, gives the
|
||||
verifier's verdict on every Batocera entry of a folder filled from
|
||||
the collection."""
|
||||
text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8")
|
||||
rows = [row for row in csv.DictReader(io.StringIO(text)) if row["platform_id"] == "batocera"]
|
||||
with tempfile.TemporaryDirectory(dir=TMP) as tmp:
|
||||
root = Path(tmp)
|
||||
for row in rows:
|
||||
source = REPO_ROOT / row["collection_path"] if row["collection_path"] else None
|
||||
target = root / row["destination"]
|
||||
if source and source.is_file() and not target.exists():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.link(source, target)
|
||||
except OSError:
|
||||
shutil.copyfile(source, target)
|
||||
held = [row for row in rows if (root / row["destination"]).is_file()]
|
||||
if len(held) < 100:
|
||||
raise unittest.SkipTest("the collection files are not in this checkout")
|
||||
# The verifier reads the platform's own entries, the check the CSV.
|
||||
contract = json.loads(_built(API / "platforms" / "batocera.json").read_text())[
|
||||
"items"][0]["contract"]
|
||||
declared = [entry for _sys, system in sorted(contract["systems"].items())
|
||||
for entry in system.get("files") or []]
|
||||
self.assertEqual([e["name"] for e in declared], [row["name"] for row in rows])
|
||||
mismatched = []
|
||||
zipped_ok = 0
|
||||
for row, platform_entry in zip(rows, declared):
|
||||
entry = {key: value for key, value in row.items() if value}
|
||||
path = root / row["destination"]
|
||||
ours = "ok" if check_platform_folder.accepts(entry, path, "md5") else (
|
||||
"wrong" if path.is_file() else "missing")
|
||||
theirs = _as_snippet(_verify_status(platform_entry, path, "md5"))
|
||||
if ours != theirs:
|
||||
mismatched.append((row["destination"], row.get("zipped_file"), ours, theirs))
|
||||
zipped_ok += ours == "ok" and bool(row["zipped_file"])
|
||||
self.assertEqual(mismatched, [])
|
||||
self.assertGreater(zipped_ok, 100)
|
||||
|
||||
def test_the_data_page_shows_the_script_itself(self):
|
||||
script = (REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
|
||||
page = generate_site.generate_data_page([])
|
||||
self.assertIn("```python\n" + script.rstrip("\n") + "\n```", page)
|
||||
self.assertLess(len(script.splitlines()), 70)
|
||||
self.assertIn("## Check a folder for one platform", page)
|
||||
|
||||
|
||||
class TheContractsValidate(unittest.TestCase):
|
||||
"""8.4: registry keys leave the contract, every $id is its served URL."""
|
||||
|
||||
def test_every_contract_validates_against_the_platform_schema(self):
|
||||
from jsonschema import Draft202012Validator # noqa: PLC0415
|
||||
|
||||
from common import list_registered_platforms, load_platform_config # noqa: PLC0415
|
||||
|
||||
schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text())
|
||||
validator = Draft202012Validator(schema)
|
||||
platforms = str(REPO_ROOT / "platforms")
|
||||
for name in list_registered_platforms(platforms, include_archived=True):
|
||||
config = load_platform_config(name, platforms)
|
||||
contract, registry = generate_site._split_contract(config)
|
||||
with self.subTest(platform=name):
|
||||
errors = [error.message for error in validator.iter_errors(contract)]
|
||||
self.assertEqual(errors, [])
|
||||
self.assertIn("install", registry)
|
||||
self.assertNotIn("_key", registry)
|
||||
self.assertEqual(set(contract) | set(registry) | {"_key"}, set(config) | {"_key"})
|
||||
|
||||
def test_every_schema_id_is_the_address_it_is_served_at(self):
|
||||
for path in sorted((REPO_ROOT / "schemas").glob("*.schema.json")):
|
||||
schema = json.loads(path.read_text())
|
||||
with self.subTest(schema=path.name):
|
||||
self.assertEqual(schema.get("$id"), f"{SITE}api/v1/schemas/{path.name}")
|
||||
|
||||
def test_every_schema_is_published(self):
|
||||
published = {p.name for p in _built(API / "schemas").glob("*.json")}
|
||||
self.assertEqual(published, {p.name for p in (REPO_ROOT / "schemas").glob("*.schema.json")})
|
||||
|
||||
def test_the_built_api_validates(self):
|
||||
from jsonschema import Draft202012Validator # noqa: PLC0415
|
||||
|
||||
schemas = {p.name: json.loads(p.read_text()) for p in (REPO_ROOT / "schemas").glob("*.json")}
|
||||
envelope = Draft202012Validator(schemas["site-api-envelope.schema.json"])
|
||||
platform = Draft202012Validator(schemas["platform.schema.json"])
|
||||
document = json.loads(_built(API / "platforms.json").read_text())
|
||||
self.assertEqual(list(envelope.iter_errors(document)), [])
|
||||
for item in document["items"]:
|
||||
with self.subTest(platform=item["id"]):
|
||||
self.assertEqual([e.message for e in platform.iter_errors(item["contract"])], [])
|
||||
for name in ("systems.json", "cores.json", "index.json", "catalog.json"):
|
||||
with self.subTest(document=name):
|
||||
self.assertEqual(
|
||||
[e.message for e in envelope.iter_errors(json.loads((API / name).read_text()))], [])
|
||||
|
||||
|
||||
class EveryEnvelopeCarriesTheCommit(unittest.TestCase):
|
||||
"""8.5: emulators/ and systems/ lacked `commit` (716 documents)."""
|
||||
|
||||
def test_the_writers_pass_it(self):
|
||||
source = Path(generate_site.__file__).read_text(encoding="utf-8")
|
||||
self.assertIn('str(db.get("generated_at") or _timestamp()), _source_commit(),\n )\n'
|
||||
" clash", source)
|
||||
block = source[source.index("public_profiles = {"):source.index("pages = emulator_pages(")]
|
||||
self.assertIn("write_single_item(", block)
|
||||
self.assertIn("_source_commit()", block)
|
||||
|
||||
def test_every_built_envelope_has_it(self):
|
||||
_built(API / "catalog.json")
|
||||
if not generate_site._source_commit():
|
||||
raise unittest.SkipTest("not a git checkout")
|
||||
lacking = [
|
||||
path.relative_to(API).as_posix() for path in sorted(API.rglob("*.json"))
|
||||
if path.parent.name not in ("schemas", "manifests")
|
||||
and path.name not in ("database.json", "stats.json", "release.json")
|
||||
and "commit" not in json.loads(path.read_text())
|
||||
]
|
||||
self.assertEqual(lacking, [])
|
||||
|
||||
|
||||
class ManifestsArePublished(unittest.TestCase):
|
||||
"""8.2: the manifest a pack installs, byte for byte, in the catalog."""
|
||||
|
||||
def test_each_manifest_is_the_repository_file(self):
|
||||
catalog = json.loads(_built(API / "catalog.json").read_text())
|
||||
listed = {item["url"]: item for item in catalog["items"]}
|
||||
for source in sorted((REPO_ROOT / "install").glob("*.json")):
|
||||
with self.subTest(manifest=source.name):
|
||||
item = listed[f"manifests/{source.name}"]
|
||||
self.assertEqual(item["sha256"], hashlib.sha256(source.read_bytes()).hexdigest())
|
||||
self.assertEqual(item["schema"], "schemas/install-manifest.schema.json")
|
||||
self.assertIn("release.json", listed)
|
||||
self.assertEqual(catalog["license"], generate_site.DATA_LICENSE)
|
||||
|
||||
def test_the_data_page_explains_the_fields_from_the_schema(self):
|
||||
page = generate_site.generate_data_page([])
|
||||
schema = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text())
|
||||
for field in ("dest", "repo_path", "release_asset", "cores"):
|
||||
description = schema["$defs"]["file"]["properties"][field]["description"]
|
||||
self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page)
|
||||
self.assertIn("## Platform manifests", page)
|
||||
self.assertIn("**Licence:**", page)
|
||||
self.assertIn("## Hosting and limits", page)
|
||||
# The endpoints first, the DAT below them.
|
||||
self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML"))
|
||||
|
||||
|
||||
class AnIdLeadsToItsData(unittest.TestCase):
|
||||
"""8.6: a system by its id, a core by the name the buildbot gives it."""
|
||||
|
||||
def test_a_buildbot_name_designates_its_profile(self):
|
||||
profiles = {
|
||||
"beetle_psx": {"type": "libretro", "cores": ["mednafen_psx", "mednafen_psx_hw"]},
|
||||
"pcsx2": {"type": "standalone"},
|
||||
"lrps2": {"type": "libretro", "cores": ["pcsx2"]},
|
||||
"mednafen_psx_alias": {"type": "alias", "alias_of": "beetle_psx"},
|
||||
"probe": {"type": "test", "cores": ["probe_core"]},
|
||||
}
|
||||
items = generate_site.core_index_items(profiles)
|
||||
by_name: dict[str, list[tuple[str, bool]]] = {}
|
||||
for item in items:
|
||||
by_name.setdefault(item["name"], []).append((item["profile"], item["preferred"]))
|
||||
self.assertEqual(by_name["mednafen_psx"], [("beetle_psx", True)])
|
||||
self.assertEqual(by_name["pcsx2"], [("pcsx2", True), ("lrps2", False)])
|
||||
self.assertEqual(by_name["mednafen_psx_alias"], [("beetle_psx", True)])
|
||||
self.assertNotIn("probe_core", by_name)
|
||||
|
||||
def test_the_built_index_resolves_ids_to_sections(self):
|
||||
items = {item["id"]: item for item in
|
||||
json.loads(_built(API / "systems.json").read_text())["items"]}
|
||||
color = items["wonderswan-color"]
|
||||
self.assertEqual(color["name"], "Bandai WonderSwan Color")
|
||||
self.assertTrue(color["url"].startswith("/retrobios/systems/bandai/"), color["url"])
|
||||
self.assertTrue(items["sony-playstation"]["sha1"])
|
||||
cores = json.loads((API / "cores.json").read_text())["items"]
|
||||
self.assertIn({"name": "mednafen_psx", "profile": "beetle_psx", "preferred": True}, cores)
|
||||
|
||||
|
||||
class LookupUrlsResolve(unittest.TestCase):
|
||||
"""8.7: a record's url, joined to the shard holding it, is a page."""
|
||||
|
||||
def test_site_absolute(self):
|
||||
db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin"}}}
|
||||
[item] = generate_site.file_location_items(db)
|
||||
self.assertTrue(item["url"].startswith("/retrobios/systems/"), item["url"])
|
||||
shard = SITE + "api/v1/lookup/ab.json"
|
||||
self.assertTrue(urllib.parse.urljoin(shard, item["url"]).startswith(SITE + "systems/"))
|
||||
|
||||
def test_each_record_names_its_system_as_the_pages_do(self):
|
||||
"""Find a file opens on "a BIOS file for Sony PlayStation": the name
|
||||
comes from system_name, never built in the browser."""
|
||||
from siterender import system_name # noqa: PLC0415
|
||||
|
||||
db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin",
|
||||
"name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}}}
|
||||
coverages = {"batocera": {"platform": "Batocera", "config": {"systems": {
|
||||
"sony-playstation": {"files": [{"name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}]}}}}}
|
||||
profiles = {"duckstation": {"systems": ["sony-playstation"], "files": [
|
||||
{"name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}]}}
|
||||
records = generate_site.file_lookup_records(
|
||||
db, coverages, profiles, generate_site.file_location_items(db))
|
||||
expected = system_name("sony-playstation")
|
||||
by_layer = {r["layer"]: r for r in records if r["algorithm"] == "md5"}
|
||||
for layer in ("collection", "platform", "emulator"):
|
||||
with self.subTest(layer=layer):
|
||||
self.assertEqual(by_layer[layer]["system_name"], expected)
|
||||
|
||||
def test_built_shards_carry_the_name(self):
|
||||
shard = json.loads(_built(API / "lookup" / "05.json").read_text())
|
||||
record = next(r for r in shard["items"] if r["value"].startswith("0555c6fae890"))
|
||||
self.assertEqual(record["system_name"], "Sony PlayStation")
|
||||
|
||||
def test_built_records_land_on_built_pages(self):
|
||||
shard = json.loads(_built(API / "lookup" / "05.json").read_text())
|
||||
dist = REPO_ROOT / "website" / "dist"
|
||||
for record in shard["items"][:200]:
|
||||
with self.subTest(url=record["url"]):
|
||||
self.assertTrue(record["url"].startswith("/retrobios/"))
|
||||
if dist.is_dir():
|
||||
page = urllib.parse.unquote(record["url"].split("#")[0].removeprefix("/retrobios/"))
|
||||
self.assertTrue((dist / page / "index.html").is_file(), page)
|
||||
|
||||
|
||||
class PlatformPagesLinkTheirSystemsAndEmulators(unittest.TestCase):
|
||||
"""8.9: the overview named emulators as ids, none of 1,463 linked."""
|
||||
|
||||
def test_an_emulator_is_named_and_linked(self):
|
||||
link = generate_site._emulator_named_link("beetle_psx", "../")
|
||||
self.assertRegex(link, r"^\[[^\]]+\]\(\.\./emulators/beetle_psx\.md\)$")
|
||||
self.assertNotEqual(link, "[beetle_psx](../emulators/beetle_psx.md)")
|
||||
|
||||
def test_the_overview_column_uses_it(self):
|
||||
source = Path(generate_site.__file__).read_text(encoding="utf-8")
|
||||
block = source[source.index("### Systems overview"):source.index("_render_platform_systems(by_system")]
|
||||
self.assertIn("_emulator_named_link(e, \"../\")", block)
|
||||
self.assertNotIn('", ".join(sys_emus[:3])', block)
|
||||
|
||||
def test_a_moved_list_opens_on_its_console_and_names_its_parent(self):
|
||||
# Past PLATFORM_SECTION_BUDGET, so the list leaves for a page of its own.
|
||||
files = [{"name": f"{'f' * 200}{i}.bin", "status": "ok"} for i in range(1200)]
|
||||
moved: dict[str, str] = {}
|
||||
generate_site._render_platform_systems(
|
||||
{"sony-playstation": files}, {}, {}, ("retroarch", "RetroArch"), moved, {})
|
||||
self.assertTrue(moved)
|
||||
page = next(iter(moved.values()))
|
||||
self.assertTrue(page.startswith('---\nparent:\n label: "RetroArch"\n'
|
||||
' link: "/platforms/retroarch/"\n---\n'), page[:120])
|
||||
self.assertRegex(page, r"RetroArch's list for \[Sony PlayStation\]\(\.\./systems/")
|
||||
|
||||
def test_part_pages_of_gaps_and_cross_reference_name_their_parent(self):
|
||||
moved: dict[str, str] = {}
|
||||
generate_site._list_pages("gaps-x", "T", "intro", [], [["| a |"]], moved,
|
||||
parent=("What the packs do not hold", "/gaps/"))
|
||||
self.assertIn('parent:\n label: "What the packs do not hold"\n link: "/gaps/"', moved["gaps-x"])
|
||||
source = Path(generate_site.__file__).read_text(encoding="utf-8")
|
||||
self.assertIn('_parent_front_matter(("Cores by platform", "/cross-reference/"))', source)
|
||||
self.assertIn('parent=("Dump provenance", "/provenance/")', source)
|
||||
|
||||
|
||||
class TablesFitTheirColumn(unittest.TestCase):
|
||||
"""8.12 and 8.13: a hash in a cell is 12 characters, the rest in its title."""
|
||||
|
||||
def test_the_downloads_table(self):
|
||||
item = {"title": "SQLite snapshot", "url": "downloads/retrobios.sqlite",
|
||||
"media_type": "application/vnd.sqlite3", "bytes": 10, "sha256": "a" * 64}
|
||||
page = generate_site.generate_data_page([item])
|
||||
row = next(line for line in page.splitlines() if "SQLite snapshot" in line and "|" in line)
|
||||
self.assertIn(f'<code title="{"a" * 64}">{"a" * 12}</code>', row)
|
||||
self.assertNotIn("a" * 13, row.replace(f'title="{"a" * 64}"', ""))
|
||||
|
||||
def test_the_gaps_tables(self):
|
||||
problem = {"platform": "Batocera", "name": "x.bin", "status": "untested", "required": True,
|
||||
"reason": "expected md5 differs", "declared": {"md5": ["b" * 32]}, "path": ""}
|
||||
table = "\n".join(generate_site._render_problem_files([problem], db={"files": {}}))
|
||||
row = next(line for line in table.splitlines() if "x.bin" in line)
|
||||
self.assertIn(f'<code title="{"b" * 32}">{"b" * 12}</code>', row)
|
||||
self.assertIn('<span class="rb-reason">expected md5 differs</span>', row)
|
||||
source = Path(generate_site.__file__).read_text(encoding="utf-8")
|
||||
self.assertIn('expected = f"{algorithm} {short_hash(value)}"', source)
|
||||
self.assertIn("{_reason_cell(u['reason'])}", source)
|
||||
|
||||
|
||||
class SourceLinksGoWhereTheFileIs(unittest.TestCase):
|
||||
"""8.14: trident cited a Panda3DS path and linked it into 3dsTrident."""
|
||||
|
||||
PROFILE = {
|
||||
"emulator": "trident", "type": "libretro",
|
||||
"source": "https://github.com/DanAlexMorton/3dsTrident",
|
||||
"upstream": "https://github.com/wheremyfoodat/Panda3DS",
|
||||
"source_commit": "b" * 40,
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
sitesources._tree_paths.cache_clear()
|
||||
|
||||
def _render(self, tree: frozenset[str] | None) -> str:
|
||||
original = sitesources._tree_paths
|
||||
sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment]
|
||||
try:
|
||||
return sitesources._source_ref_markdown(
|
||||
self.PROFILE, "src/emulator.cpp:234; src/core/loader/ncch.cpp:151-164")
|
||||
finally:
|
||||
sitesources._tree_paths = original
|
||||
|
||||
def test_a_path_the_tree_lacks_is_not_linked(self):
|
||||
text = self._render(frozenset({"src/emulator.cpp"}))
|
||||
self.assertIn("[`src/emulator.cpp:234`](https://github.com/DanAlexMorton/3dsTrident/", text)
|
||||
self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
|
||||
self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text)
|
||||
|
||||
def test_without_a_cached_tree_the_declarations_decide(self):
|
||||
text = self._render(None)
|
||||
self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text)
|
||||
|
||||
def test_every_rendered_link_is_in_its_cached_tree(self):
|
||||
"""Offline: each link the pages render points at a path the cached
|
||||
tree of its repository holds."""
|
||||
if not sitesources.TREE_CACHE.is_dir():
|
||||
raise unittest.SkipTest("no cached tree listings")
|
||||
from common import load_emulator_profiles # noqa: PLC0415
|
||||
|
||||
link = re.compile(r"\(https://github\.com/([^/]+/[^/]+)/blob/([0-9a-f]{40})/([^#)]+)")
|
||||
checked, wrong = 0, []
|
||||
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False)
|
||||
for key, profile in sorted(profiles.items()):
|
||||
for entry in profile.get("files") or []:
|
||||
if not isinstance(entry, dict) or not entry.get("source_ref"):
|
||||
continue
|
||||
for slug, pin, path in link.findall(
|
||||
sitesources._source_ref_markdown(profile, entry["source_ref"])):
|
||||
import upstream # noqa: PLC0415
|
||||
|
||||
repo = upstream.parse_repo(f"https://github.com/{slug}")
|
||||
held = sitesources.tree_holds(repo, pin, urllib.parse.unquote(path))
|
||||
if held is None:
|
||||
continue
|
||||
checked += 1
|
||||
if not held:
|
||||
wrong.append((key, slug, path))
|
||||
if not checked:
|
||||
raise unittest.SkipTest("no rendered link has a cached tree")
|
||||
self.assertEqual(wrong, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -224,7 +224,7 @@ class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase):
|
||||
members = {"freeintv-overlays": {"a.png": {"sha1": "02" * 20}}}
|
||||
items = generate_site.file_location_items({"files": {}}, self.PROFILES, members)
|
||||
self.assertEqual(items, [{"sha1": "02" * 20,
|
||||
"url": "emulators/freeintv/#sha1-020202020202"}])
|
||||
"url": "/retrobios/emulators/freeintv/#sha1-020202020202"}])
|
||||
|
||||
def test_the_parent_page_lists_the_tree(self):
|
||||
import generate_site
|
||||
|
||||
@@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` |
|
||||
| Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
|
||||
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
|
||||
## Test architecture
|
||||
|
||||
|
||||
Reference in new issue
Block a user