From 6d938f3d54767876070e372ce1dcc4908e1ef409 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:52:13 +0200 Subject: [PATCH] feat: reference pages answer every hash and field A platform or emulator record whose value designates a collection file carries its sha1 and is filed under the file's other hashes, so a SHA1 paste finds the setups listing it by MD5. The data page shows every manifest field, writes URL templates as code, states the EmuDeck rule, the browser lookup and the empty lists. systems.json gives every id holding files a page, Files to source splits at 100 rows with an upload link per file, and MiSTer and BizHawk get no empty core table. --- schemas/install-manifest.schema.json | 19 +- schemas/site-api-envelope.schema.json | 12 +- scripts/generate_site.py | 237 ++++++++++++++++-- tests/test_site_r2_reference.py | 4 +- tests/test_site_r3_reference.py | 13 +- tests/test_site_r4_reference.py | 344 ++++++++++++++++++++++++++ 6 files changed, 591 insertions(+), 38 deletions(-) create mode 100644 tests/test_site_r4_reference.py diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index e3587ce6..62952afd 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -23,21 +23,24 @@ ], "properties": { "manifest_version": { - "const": 2 + "const": 2, + "description": "Format version of the manifest. install.py refuses a version it does not know." }, "source": { "enum": [ "full", "platform", "truth" - ] + ], + "description": "Which files the pack was built from: full (the platform's list plus the files its cores load), platform (the list alone) or truth (the cores' files alone)." }, "regions": { "type": "array", "items": { "type": "string" }, - "uniqueItems": true + "uniqueItems": true, + "description": "Region priority the manifest was narrowed to, best first. Empty when it was not narrowed." }, "target": { "type": "string", @@ -79,13 +82,15 @@ "type": "array", "items": { "type": "object" - } + }, + "description": "Rules install.py follows to find this setup and its BIOS folder on a machine, one per method and operating system." }, "standalone_copies": { "type": "array", "items": { "type": "object" - } + }, + "description": "Files install.py also copies into the folders of standalone emulators it finds, or a note it prints when their folder exists." }, "total_files": { "type": "integer", @@ -497,7 +502,7 @@ }, "repo_path": { "type": "string", - "description": "Path in the repository. Fetched from https://raw.githubusercontent.com/Abdess/retrobios//, ref being main unless RETROBIOS_REF names a tag." + "description": "Path in the repository. Fetched from `https://raw.githubusercontent.com/Abdess/retrobios//`, ref being main unless RETROBIOS_REF names a tag." }, "cores": { "oneOf": [ @@ -521,7 +526,7 @@ "release_asset": { "type": "string", "pattern": "^[^/\\\\]+$", - "description": "Asset of the large-files release, fetched from https://github.com/Abdess/retrobios/releases/download/large-files/ instead of repo_path." + "description": "Asset of the large-files release, fetched from `https://github.com/Abdess/retrobios/releases/download/large-files/` instead of repo_path." }, "systems": { "description": "Every platform system that declares the file, for the installer's --system.", diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 5372333e..63f7b5ca 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -403,7 +403,17 @@ }, "sha1": { "type": "string", - "pattern": "^[0-9a-f]{40}$" + "pattern": "^[0-9a-f]{40}$", + "description": "SHA1 of the collection file the value designates. A platform or emulator record that designates no collection file has none." + }, + "declared": { + "enum": [ + "sha1", + "md5", + "crc32", + "sha256" + ], + "description": "On a copy of a platform or emulator record filed under another hash of its file: the algorithm of the value its owner wrote." }, "owner": { "type": "string" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 446e91a3..587d66f4 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1758,8 +1758,11 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: api/v1/systems/ is keyed by manufacturer and its consoles by display name: a frontend keying by system id (sony-playstation) had no way in. The url is the section system_section gives, the one every page links. + An id with no section of its own (tyrian, scummvm) that holds files + takes the card of its first file, else the page of the emulator naming + it: 28 such ids sent their hashes to no page. """ - from siterender import system_ids, system_section + from siterender import card_page, system_ids, system_section sha1_by_path = _sha1_by_path(db) files = db.get("files") or {} @@ -1795,16 +1798,33 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: name = system_name(sys_id) except KeyError: name = sys_id - items.append({ - "id": sys_id, - "name": name, - "manufacturer": manufacturers.get(section[0], "") if section else "", - "url": _site_path(console_href(*section)) if section else None, - "sha1": sorted(held.get(sys_id, set()) & files.keys()), - }) + sha1s = sorted(held.get(sys_id, set()) & files.keys()) + manufacturer = manufacturers.get(section[0], "") if section else "" + url = _site_path(console_href(*section)) if section else None + if url is None and sha1s: + manufacturer, url = _first_file_place(sys_id, sha1s, files, profiles, card_page) + items.append({"id": sys_id, "name": name, "manufacturer": manufacturer, + "url": url, "sha1": sha1s}) return items +def _first_file_place( + sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page, +) -> tuple[str, str | None]: + """(manufacturer, url) of the card of an id's first collection file, + else of the page of the first emulator naming the id.""" + for sha1 in sha1s: + parts = str(files[sha1].get("path", "")).split("/") + if len(parts) >= 3 and parts[0] == "bios": + page = card_page("/".join(parts), sha1) + return parts[1], _site_path(f"systems/{page}.md#sha1-{sha1[:12]}") + for key in sorted(profiles): + profile = profiles[key] + if profile.get("type") not in ("alias", "test") and sys_id in (profile.get("systems") or []): + return "", _site_path(f"emulators/{key}.md") + return "", None + + def core_index_items(profiles: dict) -> list[dict]: """Every name a core goes by, with the profile it designates. @@ -1834,14 +1854,28 @@ def _is_manifest(item: dict) -> bool: return item["url"].startswith("api/v1/manifests/") +# A URL template written in prose (https://...//): escaped, +# the autolinker made a link of it, <ref> on screen and a 404 behind it. +_URL_TEMPLATE = re.compile(r"(? list[str]: """One table row per field, its description read from the schema, so the - page and the published contract say the same thing.""" + page and the published contract say the same thing. A URL template is + written as code.""" props = schema.get("properties") or {} - return [ - f"| `{key}` | {_escape_tags(str(props[key]['description']))} |" - for key in keys if (props.get(key) or {}).get("description") - ] + rows = [] + for key in keys: + text = str((props.get(key) or {}).get("description") or "") + if text: + rows.append(f"| `{key}` | {_escape_tags(_URL_TEMPLATE.sub(r'`\g<0>`', text))} |") + return rows + + +def _other_fields(schema: dict, shown: tuple[str, ...]) -> list[str]: + """The fields a short table leaves out, every one with its description.""" + rest = tuple(key for key in schema.get("properties") or {} if key not in shown) + return ["| Field | Meaning |", "|-------|---------|", *_schema_field_rows(schema, rest)] # The manifest fields an integrator reads first, in the order they matter. @@ -1859,12 +1893,34 @@ def _short_sha256(item: dict, catalog: str) -> str: return f'{value[:12]}' +def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]: + """The names of the platforms whose registry says they keep no list.""" + platforms_dir = platforms_dir or _REPO_ROOT / "platforms" + names = [] + for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()): + if entry.get("native_list") != "none": + continue + config = platforms_dir / str(entry.get("config") or f"{key}.yml") + data = yaml_load(config.read_text(encoding="utf-8")) if config.is_file() else {} + names.append(str((data or {}).get("platform") or key)) + return names + + +def _join_names(names: list[str]) -> str: + return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}" + + def generate_data_page(exports: list[dict]) -> str: manifest_schema = json.loads( (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") manifests = [item for item in exports if _is_manifest(item)] + no_list = _platforms_without_a_list() lines = [ + "---", + "terms: manifest, schema, check a folder, rate limit, api", + "---", + "", "# Data and API", "", "RetroBIOS publishes the same metadata used by its verifier, pack builder " @@ -1907,6 +1963,10 @@ def generate_data_page(exports: list[dict]) -> str: "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " "Each schema's `$id` is the address it is served at.", "", + "`database.json`, `stats.json`, `release.json` and the manifests are not " + "envelopes. Their revision is the `commit` of `catalog.json` that lists " + "their SHA256.", + "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " @@ -1923,7 +1983,10 @@ def generate_data_page(exports: list[dict]) -> str: "`url` of the section holding its files and their SHA1. " "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " - "whose document is `api/v1/emulators/.json`.", + "whose document is `api/v1/emulators/.json`. A name two profiles " + "claim (`pcsx2`) has one item per profile, and `preferred: true` marks the " + "one a platform list naming that core gets: the profile with that very id, " + "else the first id in alphabetical order.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " @@ -1931,7 +1994,27 @@ def generate_data_page(exports: list[dict]) -> str: "(the collection, a data directory, a platform's list or an emulator " "profile), the file `name`, the `url` of the page that shows it and, " "when one is known, the `system_name` the site gives its system. " - "[Find a file](files/index.md) reads it.", + "[Find or check a file](files/index.md) reads it.", + "", + "In a browser, the SHA1 of a file the reader picks finds its records. Web " + "Crypto has no MD5, and none is needed: a setup that lists a file by MD5 " + "or CRC32 is filed under the SHA1 of the collection file it names too.", + "", + "```js", + "// The records of the file a reader picked, by its SHA1.", + "async function lookUp(file) {", + " const digest = await crypto.subtle.digest(\"SHA-1\", await file.arrayBuffer());", + " const sha1 = [...new Uint8Array(digest)]", + " .map((byte) => byte.toString(16).padStart(2, \"0\"))", + " .join(\"\");", + f" const base = \"{SITE_URL}api/v1/lookup/\";", + " const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);", + " if (!response.ok) return [];", + " const shard = await response.json();", + " // layer: collection, data-directory, platform or emulator. owner: which one.", + " return shard.items.filter((item) => item.value === sha1);", + "}", + "```", "", "Two kinds of `url`:", "", @@ -1965,6 +2048,15 @@ def generate_data_page(exports: list[dict]) -> str: "|-------|---------|", *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", + *markup.details("note", "All fields", [ + "The manifest:", + "", + *_other_fields(manifest_schema, MANIFEST_TOP_FIELDS), + "", + "Each entry of `files`:", + "", + *_other_fields(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), + ]), "## Check a folder for one platform", "", "The job most tools come for: is this BIOS folder what the platform checks? " @@ -1979,8 +2071,12 @@ def generate_data_page(exports: list[dict]) -> str: "Recalbox writes several accepted md5 in one field, comma separated.", "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " "read in name order, never the archive's.", + "- An entry with no `destination` (EmuDeck) names no file: the system " + "passes when any file in the BIOS folder has one of its md5.", "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " "The script below does not apply it.", + *([f"- {_join_names(no_list)} have no list of their own (`native_list: none`). " + "An empty list does not mean nothing is needed."] if no_list else []), "", "```python", snippet.rstrip("\n"), @@ -2057,13 +2153,17 @@ def generate_data_page(exports: list[dict]) -> str: "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " "own file does not carry.", "", - "## Hosting and limits", + "## Hosting, rate limits and caching", "", "- Static files on GitHub Pages, read with a plain GET. No key, no account.", "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " "browser tool can call them directly.", "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", + "- GitHub may answer HTTP 429 to a client that sends too many requests, and " + "a Pages site has a soft limit of 100 GB of traffic a month ([GitHub Pages " + "limits](https://docs.github.com/en/pages/getting-started-with-github-pages/" + "github-pages-limits)). Fetch the one shard or document a question needs.", "- The base is the project's address, " f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", "", @@ -3696,6 +3796,8 @@ def generate_platform_page( if no_list: lines.extend(_core_folders_section(name, config)) + if _own_list_only(config): + lines.extend([f"**Cores:** {OWN_LIST_ONLY}", ""]) # System summary table (quick navigation) lines.extend( @@ -3726,9 +3828,12 @@ def generate_platform_page( if len(sys_emus) > 3: emu_str += f" [+{len(sys_emus) - 3}](#{anchor})" # The name a reader filters on, the id the list writes beside it. + # The id and the count are no words a reader searches for: indexed, + # every platform page answered a system id before its system's page. lines.append( - f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} " - f"| {ok_count}/{total} | {status} | {emu_str} |" + f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) " + f"{sys_id} " + f"| {ok_count}/{total} | {status} | {emu_str} |" ) lines.append("") @@ -3752,7 +3857,13 @@ def generate_files_page() -> str: address and opens the card directly. """ return "\n".join([ - "# Find a file", + "---", + "terms: check dump, md5 check, verify bios file, check hash", + "---", + "", + "# Find or check a file", + "", + "Check a BIOS dump: paste its hash or choose the file.", "", "A hash is a fingerprint of a file's contents ([Glossary](../wiki/glossary.md)). " "Tools such as clrmamepro and RomVault print one, and so do emulator logs " @@ -3785,7 +3896,7 @@ def generate_files_page() -> str: "## The index behind it", "", "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " - "first two characters of the hash, about 20 KB compressed. " + "first two characters of the hash: one of 256, never the whole index. " "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " "card by SHA1 in one file.", "", @@ -3988,11 +4099,46 @@ def file_lookup_records( "owner": key, "name": name, "url": url, **({"system_name": entry_label} if entry_label else {}), }) + records += _held_under_every_hash(records, files, db.get("indexes") or {}) unique = {json.dumps(r, sort_keys=True): r for r in records} return sorted(unique.values(), key=lambda r: ( r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) +def _collection_sha1(files: dict, indexes: dict, algorithm: str, value: str) -> str | None: + """The collection file a declared value designates, None when none does.""" + if algorithm == "sha1": + return value if value in files else None + found = indexes.get(f"by_{algorithm}", {}).get(value) + if isinstance(found, list): + found = found[0] if len(found) == 1 else None + return found if found in files else None + + +def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> list[dict]: + """A platform or emulator record whose value designates a collection + file, written under the file's other hashes too. + + Batocera lists scph5501.bin by md5: pasted by its sha1, the file showed + no setup at all. Each record now carries the file's sha1, and + ``declared`` names the hash its owner wrote when the record is a copy. + """ + copies: list[dict] = [] + for record in records: + if record["layer"] not in ("platform", "emulator"): + continue + sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"]) + if sha1 is None: + continue + record["sha1"] = sha1 + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(files[sha1].get(algorithm)): + if value != record["value"]: + copies.append({**record, "value": value, "algorithm": algorithm, + "declared": record["algorithm"]}) + return copies + + def write_lookup_shards( api: Path, records: list[dict], generated_at: str, commit: str = "", ) -> int: @@ -7478,6 +7624,26 @@ def _ratio_cell(done: int, total: int, alarm: int = 0) -> str: return cell +# GitHub's upload page for a folder of bios/: a browser upload by someone +# without write access lands on a fork and opens a pull request. +UPLOAD_URL = f"{REPO_URL}/upload/main/bios" +# Rows of Files to source on one page: 601 on one page made a phone page +# 274,509 px long. +FILES_TO_SOURCE_PART = 100 + + +def _upload_folder(profile: dict) -> str: + """The bios/ folder a file the emulator loads is uploaded into: the + console folder of its first system that has one, else none.""" + from siterender import _own_section + + for sys_id in profile.get("systems") or []: + own = _own_section(str(sys_id)) + if own is not None: + return "/".join(own) + return "" + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -7891,9 +8057,11 @@ def generate_gap_analysis( f"[{_escape_tags(display)}](emulators/{key}.md)" for display, key in zip(m["emulators"], m["emu_keys"]) ) + folder = _upload_folder(profiles[m["emu_keys"][0]]) + upload = f"{UPLOAD_URL}/{urllib.parse.quote(folder)}" if folder else UPLOAD_URL rows.append( - f"| `{m['name']}` | {destination} | {expected} | {emulators} | {req} | " - f"{plat_badges} | " + f"| `{m['name']}` [Have it? Upload]({upload}) | {destination} | {expected} " + f"| {emulators} | {req} | {plat_badges} | " f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |" ) lines.extend(["### Files to source", "", counted, ""]) @@ -7905,7 +8073,8 @@ def generate_gap_analysis( f"{counted} Each is a file an emulator loads that no platform lists and " "the collection does not hold. Part of " "[What the packs do not hold](gaps.md#files-to-source).", - head, [[row] for row in rows], moved, + head, [rows[i:i + FILES_TO_SOURCE_PART] for i in range( + 0, len(rows), FILES_TO_SOURCE_PART)], moved, budget=1, parent=("What the packs do not hold", "/gaps/"), ) lines.extend([f"Listed on {where}.", ""]) @@ -8082,6 +8251,16 @@ def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]: return len(matched), rows +# What a platform that declares no core (cores: []) is told, on its page +# and in the cross-reference. +OWN_LIST_ONLY = "None. Its own list is the whole truth: no core profile adds a file." + + +def _own_list_only(config: dict) -> bool: + """The platform declares `cores: []`: its own list decides alone.""" + return isinstance(config.get("cores"), list) and not config["cores"] + + def _render_xref_by_platform( coverages: dict, unique: dict, moved: dict[str, str] | None = None, ) -> list[str]: @@ -8102,6 +8281,12 @@ def _render_xref_by_platform( config = cov["config"] count, rows = _xref_platform_rows(config, unique) stem = f"cross-reference-{pname}" + if _own_list_only(config): + # cores: [] (MiSTer, BizHawk): no profile describes what they + # read, and a page of cores would be an empty table. + lines.append(f"| [{_escape_tags(display)}](platforms/{pname}.md) | 0 " + f"| {OWN_LIST_ONLY} |") + continue lines.append( f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}" f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |" @@ -9180,6 +9365,12 @@ def _generate(args: argparse.Namespace) -> None: ) for stem, page in xref_pages.items(): write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) + for pname, cov in coverages.items(): + if _own_list_only(cov["config"]): + # The empty table it had is gone: its address leads to the + # platform's page, which says its own list decides alone. + sitestarlight.MOVED_PAGES.setdefault( + f"cross-reference-{pname}.md", f"platforms/{pname}.md") print("Generating gap analysis page...") gap_pages: dict[str, str] = {} diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py index 9234398c..44f3cea9 100644 --- a/tests/test_site_r2_reference.py +++ b/tests/test_site_r2_reference.py @@ -208,7 +208,7 @@ class PlatformReferenceInWords(unittest.TestCase): systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}} page = self._page("md5", "md5", systems) name = siterender.system_name("sony-playstation") - self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page) + self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page) heading = next(line for line in page.splitlines() if line.startswith("### ") and "sony-playstation" in line) self.assertIn(name, heading) @@ -335,7 +335,7 @@ class DatesAndNames(unittest.TestCase): '') def test_data_page_title(self): - self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n")) + self.assertIn("\n---\n\n# Data and API\n", generate_site.generate_data_page([])) @unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(), diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py index 3741f6fe..3d5a6dec 100644 --- a/tests/test_site_r3_reference.py +++ b/tests/test_site_r3_reference.py @@ -296,7 +296,7 @@ class ManifestsArePublished(unittest.TestCase): self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page) self.assertIn("## Platform manifests", page) self.assertIn("**Licence:**", page) - self.assertIn("## Hosting and limits", page) + self.assertIn("## Hosting, rate limits and caching", page) # The endpoints first, the DAT below them. self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML")) @@ -449,7 +449,7 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase): def tearDown(self): sitesources._tree_paths.cache_clear() - def _render(self, tree: frozenset[str] | None) -> str: + def _render(self, tree: frozenset[str]) -> str: original = sitesources._tree_paths sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment] try: @@ -464,9 +464,12 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase): self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text) - def test_without_a_cached_tree_the_declarations_decide(self): - text = self._render(None) - self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text) + def test_a_path_no_tree_confirmed_is_not_linked(self): + """Round 4: without a tree the declared repository took the path, + and the deploy, which has no tree cache, linked 30 paths to a 404.""" + text = self._render(frozenset()) + self.assertNotIn("](https://", text) + self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) def test_every_rendered_link_is_in_its_cached_tree(self): """Offline: each link the pages render points at a path the cached diff --git a/tests/test_site_r4_reference.py b/tests/test_site_r4_reference.py new file mode 100644 index 00000000..722e1153 --- /dev/null +++ b/tests/test_site_r4_reference.py @@ -0,0 +1,344 @@ +"""Reference pages and data (round 4, package 8). + +The lookup answered a SHA1 without the setups that list the same file by +MD5, the published folder check failed 134 EmuDeck entries a folder EmuDeck +accepts, the manifest table printed `<ref>` behind a dead link and showed +17 of 48 fields, 28 system ids held files and no page, and source links went +to 404s on the deploy, which has no tree cache. Tests that read the built +site skip when it has not been generated. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +import tempfile +import unittest +import urllib.parse +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import check_platform_folder # noqa: E402 +import generate_site # noqa: E402 +import release_record # noqa: E402 +import sitesources # noqa: E402 +import upstream # noqa: E402 + +PUBLIC = REPO_ROOT / "website" / "public" +API = PUBLIC / "api" / "v1" +DOCS = REPO_ROOT / "website" / "src" / "content" / "docs" +DIST = REPO_ROOT / "website" / "dist" +TMP = REPO_ROOT / "tmp" + + +def _built(path: Path) -> Path: + if not path.exists(): + raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated") + return path + + +def _page(stem: str) -> str: + for suffix in (".md", ".mdx"): + path = DOCS / f"{stem}{suffix}" + if path.is_file(): + return path.read_text(encoding="utf-8") + raise unittest.SkipTest(f"{stem} not generated") + + +class LookupRecordsUnderEveryHash(unittest.TestCase): + """8.1: a platform listing a file by MD5 is found by its SHA1 too.""" + + SHA1 = "a" * 40 + FILE = {"name": "bios.bin", "path": "bios/Sony/PlayStation/bios.bin", "sha1": SHA1, + "md5": "b" * 32, "crc32": "c" * 8, "sha256": "d" * 64} + + def records(self) -> list[dict]: + db = {"files": {self.SHA1: self.FILE}, "indexes": { + "by_md5": {"b" * 32: self.SHA1}, "by_crc32": {"c" * 8: self.SHA1}, + "by_sha256": {"d" * 64: self.SHA1}}} + coverages = {"setup": {"config": {"systems": {"sony-playstation": {"files": [ + {"name": "bios.bin", "md5": "b" * 32}]}}}}} + profiles = {"core": {"type": "libretro", "systems": ["sony-playstation"], + "files": [{"name": "bios.bin", "crc32": "c" * 8}]}} + with mock.patch.object(generate_site, "_console_system_names", return_value={}), \ + mock.patch.object(generate_site, "_lookup_system_name", return_value=None), \ + mock.patch.object(generate_site, "emulator_card_page", side_effect=lambda k, a: k): + return generate_site.file_lookup_records( + db, coverages, profiles, [{"sha1": self.SHA1, "url": "/x"}]) + + def test_each_hash_names_the_setup_and_the_emulator(self): + records = self.records() + for value in (self.SHA1, "b" * 32, "c" * 8, "d" * 64): + owners = {(r["layer"], r.get("owner")) for r in records if r["value"] == value} + self.assertIn(("platform", "setup"), owners, value) + self.assertIn(("emulator", "core"), owners, value) + + def test_records_carry_the_sha1_and_say_what_was_declared(self): + for record in self.records(): + if record["layer"] in ("platform", "emulator"): + self.assertEqual(record["sha1"], self.SHA1) + copy = next(r for r in self.records() + if r["layer"] == "platform" and r["value"] == self.SHA1) + self.assertEqual(copy["declared"], "md5") + + def test_scph5501_gives_one_answer_for_its_four_hashes(self): + db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) + sha1 = "0555c6fae8906f3f09baf5988f00e55f88e9f30b" + entry = db["files"][sha1] + _built(API / "lookup") + answers = [] + for algorithm in ("sha1", "md5", "crc32", "sha256"): + value = str(entry[algorithm]).lower() + shard = json.loads((API / "lookup" / f"{value[:2]}.json").read_text()) + found = [r for r in shard["items"] if r["value"] == value] + answers.append(tuple(sorted({(r["layer"], r.get("owner", "")) for r in found}))) + self.assertEqual(len(set(answers)), 1, answers) + setups = {owner for layer, owner in answers[0] if layer == "platform"} + emulators = {owner for layer, owner in answers[0] if layer == "emulator"} + self.assertGreaterEqual(len(setups), 10) + self.assertGreaterEqual(len(emulators), 8) + + +class TheFolderCheckReadsEmuDecksWhitelist(unittest.TestCase): + """8.2: an entry with no destination names no file. EmuDeck's check + passes the system when any top-level file of the folder has its md5.""" + + def check(self, files: dict[str, bytes]) -> list[tuple[str, str]]: + good = b"ps1 bios bytes" + contract = {"verification_mode": "md5", "systems": {"sony-playstation": {"files": [ + {"name": f"sony-playstation:{hashlib.md5(good).hexdigest()}", "destination": "", + "md5": hashlib.md5(good).hexdigest()}, + {"name": "sony-playstation:" + "0" * 32, "destination": "", "md5": "0" * 32}, + ]}}} + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + root = Path(tmp) / "bios" + (root / "sub").mkdir(parents=True) + for name, data in files.items(): + (root / name).write_bytes(data) + api = Path(tmp) / "api" + api.mkdir() + (api / "emudeck.json").write_text(json.dumps({"items": [{"contract": contract}]})) + found = check_platform_folder.check("emudeck", root, api.as_uri() + "/") + # The verifier's verdict on the matching entry, file by file. + from verify import verify_entry_md5 # noqa: PLC0415 + + verdicts = {name: verify_entry_md5(contract["systems"]["sony-playstation"]["files"][0], + str(root / name))["status"] + for name in files if (root / name).is_file()} + return found, verdicts + + def test_any_name_passes(self): + found, verdicts = self.check({"my-ps1-dump.bin": b"ps1 bios bytes"}) + self.assertEqual(found, [("sony-playstation: any file", "ok")]) + self.assertEqual(verdicts, {"my-ps1-dump.bin": "ok"}) + + def test_a_folder_without_it_is_missing(self): + found, _verdicts = self.check({"other.bin": b"x", "sub/deep.bin": b"ps1 bios bytes"}) + self.assertEqual(found, [("sony-playstation: any file", "missing")]) + + def test_the_data_page_states_the_rule(self): + page = generate_site.generate_data_page([]) + self.assertIn("An entry with no `destination` (EmuDeck) names no file", page) + self.assertLess(len((REPO_ROOT / "scripts" / "check_platform_folder.py") + .read_text().splitlines()), 70) + + +class ManifestFieldsAreComplete(unittest.TestCase): + """8.3 and 8.4.""" + + SCHEMA = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()) + + def test_a_url_template_is_code(self): + rows = generate_site._schema_field_rows(self.SCHEMA["$defs"]["file"], ("repo_path",)) + self.assertIn("`https://raw.githubusercontent.com/Abdess/retrobios//`", + rows[0]) + rows = generate_site._schema_field_rows( + {"properties": {"a": {"description": "From https://x.test/, then."}}}, ("a",)) + self.assertEqual(rows, ["| `a` | From `https://x.test/`, then. |"]) + + def test_every_field_is_described_and_shown(self): + page = generate_site.generate_data_page([]) + for schema in (self.SCHEMA, self.SCHEMA["$defs"]["file"]): + for key, spec in schema["properties"].items(): + self.assertTrue(spec.get("description"), key) + self.assertIn(f"| `{key}` |", page) + + def test_no_link_carries_an_escaped_bracket(self): + html = _built(DIST / "data" / "index.html").read_text(encoding="utf-8") + self.assertNotRegex(html, r'href="[^"]*&lt;') + self.assertNotRegex(html, r'href="[^"]*<') + + +class EverySystemIdWithFilesHasAPage(unittest.TestCase): + """8.5: tyrian, scummvm, dos and 25 more had files and a null url.""" + + def test_built_index(self): + items = json.loads(_built(API / "systems.json").read_text())["items"] + orphans = [item["id"] for item in items if item["sha1"] and not item["url"]] + self.assertEqual(orphans, []) + + def test_first_file_then_emulator(self): + files = {"f" * 40: {"path": "bios/DOS/DOS/a.bin"}} + with mock.patch("siterender._card_pages", {}): + place = generate_site._first_file_place( + "dos", ["f" * 40], files, {}, lambda path, sha1: "dos") + self.assertEqual(place, ("DOS", "/retrobios/systems/dos/#sha1-ffffffffffff")) + place = generate_site._first_file_place( + "tyrian", [], files, {"opentyrian": {"systems": ["tyrian"]}}, None) + self.assertEqual(place, ("", "/retrobios/emulators/opentyrian/")) + + +# A source permalink of each forge family the pages write. +_PERMALINK = re.compile( + r"\]\(https://(?P[^/]+)/(?P[^/]+/[^/]+)/" + r"(?:blob/|-/blob/|src/commit/)(?P[0-9a-f]{40})/(?P[^#)]+)") + + +class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): + """8.6: the deploy reads the committed record, never a tree cache.""" + + def test_an_address_in_a_binary_reads_as_one(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} + text = sitesources._source_ref_markdown( + profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac") + self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, " + "offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`") + + def test_only_a_recorded_path_links(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} + repo = upstream.parse_repo("https://github.com/o/r") + record = {sitesources.tree_key(repo, "e" * 40): frozenset({"src/a.c"})} + with mock.patch.object(sitesources, "_verified", return_value=record): + sitesources._tree_paths.cache_clear() + try: + text = sitesources._source_ref_markdown(profile, "src/a.c:5, src/b.c:7") + finally: + sitesources._tree_paths.cache_clear() + self.assertIn("[`src/a.c:5`](https://github.com/o/r/blob/" + "e" * 40 + "/src/a.c#L5)", text) + self.assertIn("`src/b.c:7`", text) + self.assertNotIn("src/b.c#", text) + + def test_the_writer_keeps_held_paths_only(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40, + "notes": "see src/c.c:3", "files": [{"source_ref": "src/a.c:5, gone.c:1"}]} + with mock.patch.object(sitesources, "_cached_tree", + return_value=frozenset({"src/a.c", "src/c.c", "x.c"})): + held, unread = sitesources.build_source_paths({"p": profile}, [TMP]) + self.assertEqual(held, {"github.com/o/r@" + "e" * 40: ["src/a.c", "src/c.c"]}) + self.assertEqual(unread, []) + + def test_every_rendered_link_is_recorded(self): + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*")) + checked, stray = 0, [] + for page in pages: + for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")): + if "/Abdess/retrobios/" in found[0]: + continue + key = f"{found['host']}/{found['slug']}@{found['pin']}" + path = urllib.parse.unquote(found["path"]) + checked += 1 + if path not in record.get(key, ()): + stray.append((page.name, key, path)) + self.assertGreater(checked, 1000) + self.assertEqual(stray[:20], []) + + def test_the_record_is_read_by_the_deploy(self): + """The record is committed where the build reads it, sorted.""" + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + self.assertGreater(len(record), 300) + for key, paths in record.items(): + self.assertRegex(key, r"^[a-z0-9.-]+/.+@[0-9a-f]{40}$") + self.assertEqual(paths, sorted(set(paths)), key) + + +class TheReferencePagesAnswerTheirWords(unittest.TestCase): + """8.7, 8.8, 8.10 and 8.13.""" + + def test_find_or_check_a_file(self): + page = generate_site.generate_files_page() + self.assertTrue(page.startswith( + "---\nterms: check dump, md5 check, verify bios file, check hash\n---\n")) + self.assertIn("\n# Find or check a file\n\nCheck a BIOS dump: paste its hash " + "or choose the file.\n", page) + + def test_the_data_page(self): + page = generate_site.generate_data_page([]) + self.assertTrue(page.startswith( + "---\nterms: manifest, schema, check a folder, rate limit, api\n---\n")) + self.assertIn("## Hosting, rate limits and caching", page) + self.assertIn("HTTP 429", page) + self.assertIn("crypto.subtle.digest(\"SHA-1\"", page) + self.assertIn("api/v1/lookup/", page) + self.assertIn("`preferred: true`", page) + self.assertIn("`database.json`, `stats.json`, `release.json` and the manifests are not " + "envelopes", page) + self.assertIn("- MinUI, MyMinUI and NextUI have no list of their own " + "(`native_list: none`). An empty list does not mean nothing is needed.", page) + + def test_overview_ids_and_counts_are_not_indexed(self): + html = _built(DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8") + self.assertRegex(html, r"sony-playstation") + self.assertRegex(html, r"\d+/\d+") + + +class TheReleaseRecordHasAVersion(unittest.TestCase): + """8.9.""" + + def test_the_writer_and_the_committed_record(self): + self.assertEqual(release_record.SCHEMA_VERSION, 1) + source = (REPO_ROOT / "scripts" / "release_record.py").read_text(encoding="utf-8") + self.assertIn('{"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}', source) + schema = json.loads((REPO_ROOT / "schemas" / "release.schema.json").read_text()) + self.assertIn("schema_version", schema["required"]) + record = json.loads((REPO_ROOT / "release.json").read_text()) + self.assertEqual(record["schema_version"], 1) + + +class NoEmptyCoreTable(unittest.TestCase): + """8.11: MiSTer and BizHawk declare cores: [] and had a page of nothing.""" + + def test_own_list_platforms_get_a_sentence(self): + coverages = {"misterfpga": {"platform": "MiSTer FPGA", "config": {"cores": [], "systems": {}}}} + moved: dict[str, str] = {} + page = generate_site.generate_cross_reference(coverages, {}, moved) + self.assertEqual(moved, {}) + self.assertIn(generate_site.OWN_LIST_ONLY, page) + + def test_built_pages(self): + _built(DOCS / "cross-reference.md") + for name in ("misterfpga", "bizhawk"): + self.assertFalse((DOCS / f"cross-reference-{name}.md").exists()) + self.assertIn(generate_site.OWN_LIST_ONLY, _page(f"platforms/{name}")) + redirects = json.loads((REPO_ROOT / "website" / "src" / "generated" / + "redirects.json").read_text()) + self.assertEqual(redirects.get("/cross-reference-misterfpga/"), "/platforms/misterfpga/") + + +class FilesToSourceOnAPhone(unittest.TestCase): + """8.12: 601 rows on one page, no way to give a file.""" + + def test_parts_of_a_hundred_rows_each_with_an_upload(self): + _built(DOCS / "gaps-files-to-source.md") + parts = sorted(DOCS.glob("gaps-files-to-source*.md")) + self.assertGreater(len(parts), 1) + for part in parts: + rows = [line for line in part.read_text(encoding="utf-8").splitlines() + if line.startswith("| `")] + self.assertLessEqual(len(rows), generate_site.FILES_TO_SOURCE_PART, part.name) + for row in rows: + self.assertIn(f"[Have it? Upload]({generate_site.UPLOAD_URL}", row) + + def test_the_upload_lands_in_the_console_folder(self): + with mock.patch("siterender._own_section", return_value=("Sony", "PlayStation")): + self.assertEqual(generate_site._upload_folder({"systems": ["sony-playstation"]}), + "Sony/PlayStation") + + +if __name__ == "__main__": + unittest.main()