diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index e3587ce6..62952afd 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -23,21 +23,24 @@ ], "properties": { "manifest_version": { - "const": 2 + "const": 2, + "description": "Format version of the manifest. install.py refuses a version it does not know." }, "source": { "enum": [ "full", "platform", "truth" - ] + ], + "description": "Which files the pack was built from: full (the platform's list plus the files its cores load), platform (the list alone) or truth (the cores' files alone)." }, "regions": { "type": "array", "items": { "type": "string" }, - "uniqueItems": true + "uniqueItems": true, + "description": "Region priority the manifest was narrowed to, best first. Empty when it was not narrowed." }, "target": { "type": "string", @@ -79,13 +82,15 @@ "type": "array", "items": { "type": "object" - } + }, + "description": "Rules install.py follows to find this setup and its BIOS folder on a machine, one per method and operating system." }, "standalone_copies": { "type": "array", "items": { "type": "object" - } + }, + "description": "Files install.py also copies into the folders of standalone emulators it finds, or a note it prints when their folder exists." }, "total_files": { "type": "integer", @@ -497,7 +502,7 @@ }, "repo_path": { "type": "string", - "description": "Path in the repository. Fetched from https://raw.githubusercontent.com/Abdess/retrobios//, ref being main unless RETROBIOS_REF names a tag." + "description": "Path in the repository. Fetched from `https://raw.githubusercontent.com/Abdess/retrobios//`, ref being main unless RETROBIOS_REF names a tag." }, "cores": { "oneOf": [ @@ -521,7 +526,7 @@ "release_asset": { "type": "string", "pattern": "^[^/\\\\]+$", - "description": "Asset of the large-files release, fetched from https://github.com/Abdess/retrobios/releases/download/large-files/ instead of repo_path." + "description": "Asset of the large-files release, fetched from `https://github.com/Abdess/retrobios/releases/download/large-files/` instead of repo_path." }, "systems": { "description": "Every platform system that declares the file, for the installer's --system.", diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 5372333e..63f7b5ca 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -403,7 +403,17 @@ }, "sha1": { "type": "string", - "pattern": "^[0-9a-f]{40}$" + "pattern": "^[0-9a-f]{40}$", + "description": "SHA1 of the collection file the value designates. A platform or emulator record that designates no collection file has none." + }, + "declared": { + "enum": [ + "sha1", + "md5", + "crc32", + "sha256" + ], + "description": "On a copy of a platform or emulator record filed under another hash of its file: the algorithm of the value its owner wrote." }, "owner": { "type": "string" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 446e91a3..587d66f4 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1758,8 +1758,11 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: api/v1/systems/ is keyed by manufacturer and its consoles by display name: a frontend keying by system id (sony-playstation) had no way in. The url is the section system_section gives, the one every page links. + An id with no section of its own (tyrian, scummvm) that holds files + takes the card of its first file, else the page of the emulator naming + it: 28 such ids sent their hashes to no page. """ - from siterender import system_ids, system_section + from siterender import card_page, system_ids, system_section sha1_by_path = _sha1_by_path(db) files = db.get("files") or {} @@ -1795,16 +1798,33 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: name = system_name(sys_id) except KeyError: name = sys_id - items.append({ - "id": sys_id, - "name": name, - "manufacturer": manufacturers.get(section[0], "") if section else "", - "url": _site_path(console_href(*section)) if section else None, - "sha1": sorted(held.get(sys_id, set()) & files.keys()), - }) + sha1s = sorted(held.get(sys_id, set()) & files.keys()) + manufacturer = manufacturers.get(section[0], "") if section else "" + url = _site_path(console_href(*section)) if section else None + if url is None and sha1s: + manufacturer, url = _first_file_place(sys_id, sha1s, files, profiles, card_page) + items.append({"id": sys_id, "name": name, "manufacturer": manufacturer, + "url": url, "sha1": sha1s}) return items +def _first_file_place( + sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page, +) -> tuple[str, str | None]: + """(manufacturer, url) of the card of an id's first collection file, + else of the page of the first emulator naming the id.""" + for sha1 in sha1s: + parts = str(files[sha1].get("path", "")).split("/") + if len(parts) >= 3 and parts[0] == "bios": + page = card_page("/".join(parts), sha1) + return parts[1], _site_path(f"systems/{page}.md#sha1-{sha1[:12]}") + for key in sorted(profiles): + profile = profiles[key] + if profile.get("type") not in ("alias", "test") and sys_id in (profile.get("systems") or []): + return "", _site_path(f"emulators/{key}.md") + return "", None + + def core_index_items(profiles: dict) -> list[dict]: """Every name a core goes by, with the profile it designates. @@ -1834,14 +1854,28 @@ def _is_manifest(item: dict) -> bool: return item["url"].startswith("api/v1/manifests/") +# A URL template written in prose (https://...//): escaped, +# the autolinker made a link of it, <ref> on screen and a 404 behind it. +_URL_TEMPLATE = re.compile(r"(? list[str]: """One table row per field, its description read from the schema, so the - page and the published contract say the same thing.""" + page and the published contract say the same thing. A URL template is + written as code.""" props = schema.get("properties") or {} - return [ - f"| `{key}` | {_escape_tags(str(props[key]['description']))} |" - for key in keys if (props.get(key) or {}).get("description") - ] + rows = [] + for key in keys: + text = str((props.get(key) or {}).get("description") or "") + if text: + rows.append(f"| `{key}` | {_escape_tags(_URL_TEMPLATE.sub(r'`\g<0>`', text))} |") + return rows + + +def _other_fields(schema: dict, shown: tuple[str, ...]) -> list[str]: + """The fields a short table leaves out, every one with its description.""" + rest = tuple(key for key in schema.get("properties") or {} if key not in shown) + return ["| Field | Meaning |", "|-------|---------|", *_schema_field_rows(schema, rest)] # The manifest fields an integrator reads first, in the order they matter. @@ -1859,12 +1893,34 @@ def _short_sha256(item: dict, catalog: str) -> str: return f'{value[:12]}' +def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]: + """The names of the platforms whose registry says they keep no list.""" + platforms_dir = platforms_dir or _REPO_ROOT / "platforms" + names = [] + for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()): + if entry.get("native_list") != "none": + continue + config = platforms_dir / str(entry.get("config") or f"{key}.yml") + data = yaml_load(config.read_text(encoding="utf-8")) if config.is_file() else {} + names.append(str((data or {}).get("platform") or key)) + return names + + +def _join_names(names: list[str]) -> str: + return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}" + + def generate_data_page(exports: list[dict]) -> str: manifest_schema = json.loads( (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") manifests = [item for item in exports if _is_manifest(item)] + no_list = _platforms_without_a_list() lines = [ + "---", + "terms: manifest, schema, check a folder, rate limit, api", + "---", + "", "# Data and API", "", "RetroBIOS publishes the same metadata used by its verifier, pack builder " @@ -1907,6 +1963,10 @@ def generate_data_page(exports: list[dict]) -> str: "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " "Each schema's `$id` is the address it is served at.", "", + "`database.json`, `stats.json`, `release.json` and the manifests are not " + "envelopes. Their revision is the `commit` of `catalog.json` that lists " + "their SHA256.", + "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " @@ -1923,7 +1983,10 @@ def generate_data_page(exports: list[dict]) -> str: "`url` of the section holding its files and their SHA1. " "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " - "whose document is `api/v1/emulators/.json`.", + "whose document is `api/v1/emulators/.json`. A name two profiles " + "claim (`pcsx2`) has one item per profile, and `preferred: true` marks the " + "one a platform list naming that core gets: the profile with that very id, " + "else the first id in alphabetical order.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " @@ -1931,7 +1994,27 @@ def generate_data_page(exports: list[dict]) -> str: "(the collection, a data directory, a platform's list or an emulator " "profile), the file `name`, the `url` of the page that shows it and, " "when one is known, the `system_name` the site gives its system. " - "[Find a file](files/index.md) reads it.", + "[Find or check a file](files/index.md) reads it.", + "", + "In a browser, the SHA1 of a file the reader picks finds its records. Web " + "Crypto has no MD5, and none is needed: a setup that lists a file by MD5 " + "or CRC32 is filed under the SHA1 of the collection file it names too.", + "", + "```js", + "// The records of the file a reader picked, by its SHA1.", + "async function lookUp(file) {", + " const digest = await crypto.subtle.digest(\"SHA-1\", await file.arrayBuffer());", + " const sha1 = [...new Uint8Array(digest)]", + " .map((byte) => byte.toString(16).padStart(2, \"0\"))", + " .join(\"\");", + f" const base = \"{SITE_URL}api/v1/lookup/\";", + " const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);", + " if (!response.ok) return [];", + " const shard = await response.json();", + " // layer: collection, data-directory, platform or emulator. owner: which one.", + " return shard.items.filter((item) => item.value === sha1);", + "}", + "```", "", "Two kinds of `url`:", "", @@ -1965,6 +2048,15 @@ def generate_data_page(exports: list[dict]) -> str: "|-------|---------|", *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", + *markup.details("note", "All fields", [ + "The manifest:", + "", + *_other_fields(manifest_schema, MANIFEST_TOP_FIELDS), + "", + "Each entry of `files`:", + "", + *_other_fields(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), + ]), "## Check a folder for one platform", "", "The job most tools come for: is this BIOS folder what the platform checks? " @@ -1979,8 +2071,12 @@ def generate_data_page(exports: list[dict]) -> str: "Recalbox writes several accepted md5 in one field, comma separated.", "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " "read in name order, never the archive's.", + "- An entry with no `destination` (EmuDeck) names no file: the system " + "passes when any file in the BIOS folder has one of its md5.", "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " "The script below does not apply it.", + *([f"- {_join_names(no_list)} have no list of their own (`native_list: none`). " + "An empty list does not mean nothing is needed."] if no_list else []), "", "```python", snippet.rstrip("\n"), @@ -2057,13 +2153,17 @@ def generate_data_page(exports: list[dict]) -> str: "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " "own file does not carry.", "", - "## Hosting and limits", + "## Hosting, rate limits and caching", "", "- Static files on GitHub Pages, read with a plain GET. No key, no account.", "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " "browser tool can call them directly.", "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", + "- GitHub may answer HTTP 429 to a client that sends too many requests, and " + "a Pages site has a soft limit of 100 GB of traffic a month ([GitHub Pages " + "limits](https://docs.github.com/en/pages/getting-started-with-github-pages/" + "github-pages-limits)). Fetch the one shard or document a question needs.", "- The base is the project's address, " f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", "", @@ -3696,6 +3796,8 @@ def generate_platform_page( if no_list: lines.extend(_core_folders_section(name, config)) + if _own_list_only(config): + lines.extend([f"**Cores:** {OWN_LIST_ONLY}", ""]) # System summary table (quick navigation) lines.extend( @@ -3726,9 +3828,12 @@ def generate_platform_page( if len(sys_emus) > 3: emu_str += f" [+{len(sys_emus) - 3}](#{anchor})" # The name a reader filters on, the id the list writes beside it. + # The id and the count are no words a reader searches for: indexed, + # every platform page answered a system id before its system's page. lines.append( - f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} " - f"| {ok_count}/{total} | {status} | {emu_str} |" + f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) " + f"{sys_id} " + f"| {ok_count}/{total} | {status} | {emu_str} |" ) lines.append("") @@ -3752,7 +3857,13 @@ def generate_files_page() -> str: address and opens the card directly. """ return "\n".join([ - "# Find a file", + "---", + "terms: check dump, md5 check, verify bios file, check hash", + "---", + "", + "# Find or check a file", + "", + "Check a BIOS dump: paste its hash or choose the file.", "", "A hash is a fingerprint of a file's contents ([Glossary](../wiki/glossary.md)). " "Tools such as clrmamepro and RomVault print one, and so do emulator logs " @@ -3785,7 +3896,7 @@ def generate_files_page() -> str: "## The index behind it", "", "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " - "first two characters of the hash, about 20 KB compressed. " + "first two characters of the hash: one of 256, never the whole index. " "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " "card by SHA1 in one file.", "", @@ -3988,11 +4099,46 @@ def file_lookup_records( "owner": key, "name": name, "url": url, **({"system_name": entry_label} if entry_label else {}), }) + records += _held_under_every_hash(records, files, db.get("indexes") or {}) unique = {json.dumps(r, sort_keys=True): r for r in records} return sorted(unique.values(), key=lambda r: ( r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) +def _collection_sha1(files: dict, indexes: dict, algorithm: str, value: str) -> str | None: + """The collection file a declared value designates, None when none does.""" + if algorithm == "sha1": + return value if value in files else None + found = indexes.get(f"by_{algorithm}", {}).get(value) + if isinstance(found, list): + found = found[0] if len(found) == 1 else None + return found if found in files else None + + +def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> list[dict]: + """A platform or emulator record whose value designates a collection + file, written under the file's other hashes too. + + Batocera lists scph5501.bin by md5: pasted by its sha1, the file showed + no setup at all. Each record now carries the file's sha1, and + ``declared`` names the hash its owner wrote when the record is a copy. + """ + copies: list[dict] = [] + for record in records: + if record["layer"] not in ("platform", "emulator"): + continue + sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"]) + if sha1 is None: + continue + record["sha1"] = sha1 + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(files[sha1].get(algorithm)): + if value != record["value"]: + copies.append({**record, "value": value, "algorithm": algorithm, + "declared": record["algorithm"]}) + return copies + + def write_lookup_shards( api: Path, records: list[dict], generated_at: str, commit: str = "", ) -> int: @@ -7478,6 +7624,26 @@ def _ratio_cell(done: int, total: int, alarm: int = 0) -> str: return cell +# GitHub's upload page for a folder of bios/: a browser upload by someone +# without write access lands on a fork and opens a pull request. +UPLOAD_URL = f"{REPO_URL}/upload/main/bios" +# Rows of Files to source on one page: 601 on one page made a phone page +# 274,509 px long. +FILES_TO_SOURCE_PART = 100 + + +def _upload_folder(profile: dict) -> str: + """The bios/ folder a file the emulator loads is uploaded into: the + console folder of its first system that has one, else none.""" + from siterender import _own_section + + for sys_id in profile.get("systems") or []: + own = _own_section(str(sys_id)) + if own is not None: + return "/".join(own) + return "" + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -7891,9 +8057,11 @@ def generate_gap_analysis( f"[{_escape_tags(display)}](emulators/{key}.md)" for display, key in zip(m["emulators"], m["emu_keys"]) ) + folder = _upload_folder(profiles[m["emu_keys"][0]]) + upload = f"{UPLOAD_URL}/{urllib.parse.quote(folder)}" if folder else UPLOAD_URL rows.append( - f"| `{m['name']}` | {destination} | {expected} | {emulators} | {req} | " - f"{plat_badges} | " + f"| `{m['name']}` [Have it? Upload]({upload}) | {destination} | {expected} " + f"| {emulators} | {req} | {plat_badges} | " f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |" ) lines.extend(["### Files to source", "", counted, ""]) @@ -7905,7 +8073,8 @@ def generate_gap_analysis( f"{counted} Each is a file an emulator loads that no platform lists and " "the collection does not hold. Part of " "[What the packs do not hold](gaps.md#files-to-source).", - head, [[row] for row in rows], moved, + head, [rows[i:i + FILES_TO_SOURCE_PART] for i in range( + 0, len(rows), FILES_TO_SOURCE_PART)], moved, budget=1, parent=("What the packs do not hold", "/gaps/"), ) lines.extend([f"Listed on {where}.", ""]) @@ -8082,6 +8251,16 @@ def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]: return len(matched), rows +# What a platform that declares no core (cores: []) is told, on its page +# and in the cross-reference. +OWN_LIST_ONLY = "None. Its own list is the whole truth: no core profile adds a file." + + +def _own_list_only(config: dict) -> bool: + """The platform declares `cores: []`: its own list decides alone.""" + return isinstance(config.get("cores"), list) and not config["cores"] + + def _render_xref_by_platform( coverages: dict, unique: dict, moved: dict[str, str] | None = None, ) -> list[str]: @@ -8102,6 +8281,12 @@ def _render_xref_by_platform( config = cov["config"] count, rows = _xref_platform_rows(config, unique) stem = f"cross-reference-{pname}" + if _own_list_only(config): + # cores: [] (MiSTer, BizHawk): no profile describes what they + # read, and a page of cores would be an empty table. + lines.append(f"| [{_escape_tags(display)}](platforms/{pname}.md) | 0 " + f"| {OWN_LIST_ONLY} |") + continue lines.append( f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}" f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |" @@ -9180,6 +9365,12 @@ def _generate(args: argparse.Namespace) -> None: ) for stem, page in xref_pages.items(): write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) + for pname, cov in coverages.items(): + if _own_list_only(cov["config"]): + # The empty table it had is gone: its address leads to the + # platform's page, which says its own list decides alone. + sitestarlight.MOVED_PAGES.setdefault( + f"cross-reference-{pname}.md", f"platforms/{pname}.md") print("Generating gap analysis page...") gap_pages: dict[str, str] = {} diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py index 9234398c..44f3cea9 100644 --- a/tests/test_site_r2_reference.py +++ b/tests/test_site_r2_reference.py @@ -208,7 +208,7 @@ class PlatformReferenceInWords(unittest.TestCase): systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}} page = self._page("md5", "md5", systems) name = siterender.system_name("sony-playstation") - self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page) + self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page) heading = next(line for line in page.splitlines() if line.startswith("### ") and "sony-playstation" in line) self.assertIn(name, heading) @@ -335,7 +335,7 @@ class DatesAndNames(unittest.TestCase): '') def test_data_page_title(self): - self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n")) + self.assertIn("\n---\n\n# Data and API\n", generate_site.generate_data_page([])) @unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(), diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py index 3741f6fe..3d5a6dec 100644 --- a/tests/test_site_r3_reference.py +++ b/tests/test_site_r3_reference.py @@ -296,7 +296,7 @@ class ManifestsArePublished(unittest.TestCase): self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page) self.assertIn("## Platform manifests", page) self.assertIn("**Licence:**", page) - self.assertIn("## Hosting and limits", page) + self.assertIn("## Hosting, rate limits and caching", page) # The endpoints first, the DAT below them. self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML")) @@ -449,7 +449,7 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase): def tearDown(self): sitesources._tree_paths.cache_clear() - def _render(self, tree: frozenset[str] | None) -> str: + def _render(self, tree: frozenset[str]) -> str: original = sitesources._tree_paths sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment] try: @@ -464,9 +464,12 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase): self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text) - def test_without_a_cached_tree_the_declarations_decide(self): - text = self._render(None) - self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text) + def test_a_path_no_tree_confirmed_is_not_linked(self): + """Round 4: without a tree the declared repository took the path, + and the deploy, which has no tree cache, linked 30 paths to a 404.""" + text = self._render(frozenset()) + self.assertNotIn("](https://", text) + self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) def test_every_rendered_link_is_in_its_cached_tree(self): """Offline: each link the pages render points at a path the cached diff --git a/tests/test_site_r4_reference.py b/tests/test_site_r4_reference.py new file mode 100644 index 00000000..722e1153 --- /dev/null +++ b/tests/test_site_r4_reference.py @@ -0,0 +1,344 @@ +"""Reference pages and data (round 4, package 8). + +The lookup answered a SHA1 without the setups that list the same file by +MD5, the published folder check failed 134 EmuDeck entries a folder EmuDeck +accepts, the manifest table printed `<ref>` behind a dead link and showed +17 of 48 fields, 28 system ids held files and no page, and source links went +to 404s on the deploy, which has no tree cache. Tests that read the built +site skip when it has not been generated. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +import tempfile +import unittest +import urllib.parse +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import check_platform_folder # noqa: E402 +import generate_site # noqa: E402 +import release_record # noqa: E402 +import sitesources # noqa: E402 +import upstream # noqa: E402 + +PUBLIC = REPO_ROOT / "website" / "public" +API = PUBLIC / "api" / "v1" +DOCS = REPO_ROOT / "website" / "src" / "content" / "docs" +DIST = REPO_ROOT / "website" / "dist" +TMP = REPO_ROOT / "tmp" + + +def _built(path: Path) -> Path: + if not path.exists(): + raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated") + return path + + +def _page(stem: str) -> str: + for suffix in (".md", ".mdx"): + path = DOCS / f"{stem}{suffix}" + if path.is_file(): + return path.read_text(encoding="utf-8") + raise unittest.SkipTest(f"{stem} not generated") + + +class LookupRecordsUnderEveryHash(unittest.TestCase): + """8.1: a platform listing a file by MD5 is found by its SHA1 too.""" + + SHA1 = "a" * 40 + FILE = {"name": "bios.bin", "path": "bios/Sony/PlayStation/bios.bin", "sha1": SHA1, + "md5": "b" * 32, "crc32": "c" * 8, "sha256": "d" * 64} + + def records(self) -> list[dict]: + db = {"files": {self.SHA1: self.FILE}, "indexes": { + "by_md5": {"b" * 32: self.SHA1}, "by_crc32": {"c" * 8: self.SHA1}, + "by_sha256": {"d" * 64: self.SHA1}}} + coverages = {"setup": {"config": {"systems": {"sony-playstation": {"files": [ + {"name": "bios.bin", "md5": "b" * 32}]}}}}} + profiles = {"core": {"type": "libretro", "systems": ["sony-playstation"], + "files": [{"name": "bios.bin", "crc32": "c" * 8}]}} + with mock.patch.object(generate_site, "_console_system_names", return_value={}), \ + mock.patch.object(generate_site, "_lookup_system_name", return_value=None), \ + mock.patch.object(generate_site, "emulator_card_page", side_effect=lambda k, a: k): + return generate_site.file_lookup_records( + db, coverages, profiles, [{"sha1": self.SHA1, "url": "/x"}]) + + def test_each_hash_names_the_setup_and_the_emulator(self): + records = self.records() + for value in (self.SHA1, "b" * 32, "c" * 8, "d" * 64): + owners = {(r["layer"], r.get("owner")) for r in records if r["value"] == value} + self.assertIn(("platform", "setup"), owners, value) + self.assertIn(("emulator", "core"), owners, value) + + def test_records_carry_the_sha1_and_say_what_was_declared(self): + for record in self.records(): + if record["layer"] in ("platform", "emulator"): + self.assertEqual(record["sha1"], self.SHA1) + copy = next(r for r in self.records() + if r["layer"] == "platform" and r["value"] == self.SHA1) + self.assertEqual(copy["declared"], "md5") + + def test_scph5501_gives_one_answer_for_its_four_hashes(self): + db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) + sha1 = "0555c6fae8906f3f09baf5988f00e55f88e9f30b" + entry = db["files"][sha1] + _built(API / "lookup") + answers = [] + for algorithm in ("sha1", "md5", "crc32", "sha256"): + value = str(entry[algorithm]).lower() + shard = json.loads((API / "lookup" / f"{value[:2]}.json").read_text()) + found = [r for r in shard["items"] if r["value"] == value] + answers.append(tuple(sorted({(r["layer"], r.get("owner", "")) for r in found}))) + self.assertEqual(len(set(answers)), 1, answers) + setups = {owner for layer, owner in answers[0] if layer == "platform"} + emulators = {owner for layer, owner in answers[0] if layer == "emulator"} + self.assertGreaterEqual(len(setups), 10) + self.assertGreaterEqual(len(emulators), 8) + + +class TheFolderCheckReadsEmuDecksWhitelist(unittest.TestCase): + """8.2: an entry with no destination names no file. EmuDeck's check + passes the system when any top-level file of the folder has its md5.""" + + def check(self, files: dict[str, bytes]) -> list[tuple[str, str]]: + good = b"ps1 bios bytes" + contract = {"verification_mode": "md5", "systems": {"sony-playstation": {"files": [ + {"name": f"sony-playstation:{hashlib.md5(good).hexdigest()}", "destination": "", + "md5": hashlib.md5(good).hexdigest()}, + {"name": "sony-playstation:" + "0" * 32, "destination": "", "md5": "0" * 32}, + ]}}} + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + root = Path(tmp) / "bios" + (root / "sub").mkdir(parents=True) + for name, data in files.items(): + (root / name).write_bytes(data) + api = Path(tmp) / "api" + api.mkdir() + (api / "emudeck.json").write_text(json.dumps({"items": [{"contract": contract}]})) + found = check_platform_folder.check("emudeck", root, api.as_uri() + "/") + # The verifier's verdict on the matching entry, file by file. + from verify import verify_entry_md5 # noqa: PLC0415 + + verdicts = {name: verify_entry_md5(contract["systems"]["sony-playstation"]["files"][0], + str(root / name))["status"] + for name in files if (root / name).is_file()} + return found, verdicts + + def test_any_name_passes(self): + found, verdicts = self.check({"my-ps1-dump.bin": b"ps1 bios bytes"}) + self.assertEqual(found, [("sony-playstation: any file", "ok")]) + self.assertEqual(verdicts, {"my-ps1-dump.bin": "ok"}) + + def test_a_folder_without_it_is_missing(self): + found, _verdicts = self.check({"other.bin": b"x", "sub/deep.bin": b"ps1 bios bytes"}) + self.assertEqual(found, [("sony-playstation: any file", "missing")]) + + def test_the_data_page_states_the_rule(self): + page = generate_site.generate_data_page([]) + self.assertIn("An entry with no `destination` (EmuDeck) names no file", page) + self.assertLess(len((REPO_ROOT / "scripts" / "check_platform_folder.py") + .read_text().splitlines()), 70) + + +class ManifestFieldsAreComplete(unittest.TestCase): + """8.3 and 8.4.""" + + SCHEMA = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()) + + def test_a_url_template_is_code(self): + rows = generate_site._schema_field_rows(self.SCHEMA["$defs"]["file"], ("repo_path",)) + self.assertIn("`https://raw.githubusercontent.com/Abdess/retrobios//`", + rows[0]) + rows = generate_site._schema_field_rows( + {"properties": {"a": {"description": "From https://x.test/, then."}}}, ("a",)) + self.assertEqual(rows, ["| `a` | From `https://x.test/`, then. |"]) + + def test_every_field_is_described_and_shown(self): + page = generate_site.generate_data_page([]) + for schema in (self.SCHEMA, self.SCHEMA["$defs"]["file"]): + for key, spec in schema["properties"].items(): + self.assertTrue(spec.get("description"), key) + self.assertIn(f"| `{key}` |", page) + + def test_no_link_carries_an_escaped_bracket(self): + html = _built(DIST / "data" / "index.html").read_text(encoding="utf-8") + self.assertNotRegex(html, r'href="[^"]*&lt;') + self.assertNotRegex(html, r'href="[^"]*<') + + +class EverySystemIdWithFilesHasAPage(unittest.TestCase): + """8.5: tyrian, scummvm, dos and 25 more had files and a null url.""" + + def test_built_index(self): + items = json.loads(_built(API / "systems.json").read_text())["items"] + orphans = [item["id"] for item in items if item["sha1"] and not item["url"]] + self.assertEqual(orphans, []) + + def test_first_file_then_emulator(self): + files = {"f" * 40: {"path": "bios/DOS/DOS/a.bin"}} + with mock.patch("siterender._card_pages", {}): + place = generate_site._first_file_place( + "dos", ["f" * 40], files, {}, lambda path, sha1: "dos") + self.assertEqual(place, ("DOS", "/retrobios/systems/dos/#sha1-ffffffffffff")) + place = generate_site._first_file_place( + "tyrian", [], files, {"opentyrian": {"systems": ["tyrian"]}}, None) + self.assertEqual(place, ("", "/retrobios/emulators/opentyrian/")) + + +# A source permalink of each forge family the pages write. +_PERMALINK = re.compile( + r"\]\(https://(?P[^/]+)/(?P[^/]+/[^/]+)/" + r"(?:blob/|-/blob/|src/commit/)(?P[0-9a-f]{40})/(?P[^#)]+)") + + +class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): + """8.6: the deploy reads the committed record, never a tree cache.""" + + def test_an_address_in_a_binary_reads_as_one(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} + text = sitesources._source_ref_markdown( + profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac") + self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, " + "offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`") + + def test_only_a_recorded_path_links(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} + repo = upstream.parse_repo("https://github.com/o/r") + record = {sitesources.tree_key(repo, "e" * 40): frozenset({"src/a.c"})} + with mock.patch.object(sitesources, "_verified", return_value=record): + sitesources._tree_paths.cache_clear() + try: + text = sitesources._source_ref_markdown(profile, "src/a.c:5, src/b.c:7") + finally: + sitesources._tree_paths.cache_clear() + self.assertIn("[`src/a.c:5`](https://github.com/o/r/blob/" + "e" * 40 + "/src/a.c#L5)", text) + self.assertIn("`src/b.c:7`", text) + self.assertNotIn("src/b.c#", text) + + def test_the_writer_keeps_held_paths_only(self): + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40, + "notes": "see src/c.c:3", "files": [{"source_ref": "src/a.c:5, gone.c:1"}]} + with mock.patch.object(sitesources, "_cached_tree", + return_value=frozenset({"src/a.c", "src/c.c", "x.c"})): + held, unread = sitesources.build_source_paths({"p": profile}, [TMP]) + self.assertEqual(held, {"github.com/o/r@" + "e" * 40: ["src/a.c", "src/c.c"]}) + self.assertEqual(unread, []) + + def test_every_rendered_link_is_recorded(self): + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*")) + checked, stray = 0, [] + for page in pages: + for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")): + if "/Abdess/retrobios/" in found[0]: + continue + key = f"{found['host']}/{found['slug']}@{found['pin']}" + path = urllib.parse.unquote(found["path"]) + checked += 1 + if path not in record.get(key, ()): + stray.append((page.name, key, path)) + self.assertGreater(checked, 1000) + self.assertEqual(stray[:20], []) + + def test_the_record_is_read_by_the_deploy(self): + """The record is committed where the build reads it, sorted.""" + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + self.assertGreater(len(record), 300) + for key, paths in record.items(): + self.assertRegex(key, r"^[a-z0-9.-]+/.+@[0-9a-f]{40}$") + self.assertEqual(paths, sorted(set(paths)), key) + + +class TheReferencePagesAnswerTheirWords(unittest.TestCase): + """8.7, 8.8, 8.10 and 8.13.""" + + def test_find_or_check_a_file(self): + page = generate_site.generate_files_page() + self.assertTrue(page.startswith( + "---\nterms: check dump, md5 check, verify bios file, check hash\n---\n")) + self.assertIn("\n# Find or check a file\n\nCheck a BIOS dump: paste its hash " + "or choose the file.\n", page) + + def test_the_data_page(self): + page = generate_site.generate_data_page([]) + self.assertTrue(page.startswith( + "---\nterms: manifest, schema, check a folder, rate limit, api\n---\n")) + self.assertIn("## Hosting, rate limits and caching", page) + self.assertIn("HTTP 429", page) + self.assertIn("crypto.subtle.digest(\"SHA-1\"", page) + self.assertIn("api/v1/lookup/", page) + self.assertIn("`preferred: true`", page) + self.assertIn("`database.json`, `stats.json`, `release.json` and the manifests are not " + "envelopes", page) + self.assertIn("- MinUI, MyMinUI and NextUI have no list of their own " + "(`native_list: none`). An empty list does not mean nothing is needed.", page) + + def test_overview_ids_and_counts_are_not_indexed(self): + html = _built(DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8") + self.assertRegex(html, r"sony-playstation") + self.assertRegex(html, r"\d+/\d+") + + +class TheReleaseRecordHasAVersion(unittest.TestCase): + """8.9.""" + + def test_the_writer_and_the_committed_record(self): + self.assertEqual(release_record.SCHEMA_VERSION, 1) + source = (REPO_ROOT / "scripts" / "release_record.py").read_text(encoding="utf-8") + self.assertIn('{"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}', source) + schema = json.loads((REPO_ROOT / "schemas" / "release.schema.json").read_text()) + self.assertIn("schema_version", schema["required"]) + record = json.loads((REPO_ROOT / "release.json").read_text()) + self.assertEqual(record["schema_version"], 1) + + +class NoEmptyCoreTable(unittest.TestCase): + """8.11: MiSTer and BizHawk declare cores: [] and had a page of nothing.""" + + def test_own_list_platforms_get_a_sentence(self): + coverages = {"misterfpga": {"platform": "MiSTer FPGA", "config": {"cores": [], "systems": {}}}} + moved: dict[str, str] = {} + page = generate_site.generate_cross_reference(coverages, {}, moved) + self.assertEqual(moved, {}) + self.assertIn(generate_site.OWN_LIST_ONLY, page) + + def test_built_pages(self): + _built(DOCS / "cross-reference.md") + for name in ("misterfpga", "bizhawk"): + self.assertFalse((DOCS / f"cross-reference-{name}.md").exists()) + self.assertIn(generate_site.OWN_LIST_ONLY, _page(f"platforms/{name}")) + redirects = json.loads((REPO_ROOT / "website" / "src" / "generated" / + "redirects.json").read_text()) + self.assertEqual(redirects.get("/cross-reference-misterfpga/"), "/platforms/misterfpga/") + + +class FilesToSourceOnAPhone(unittest.TestCase): + """8.12: 601 rows on one page, no way to give a file.""" + + def test_parts_of_a_hundred_rows_each_with_an_upload(self): + _built(DOCS / "gaps-files-to-source.md") + parts = sorted(DOCS.glob("gaps-files-to-source*.md")) + self.assertGreater(len(parts), 1) + for part in parts: + rows = [line for line in part.read_text(encoding="utf-8").splitlines() + if line.startswith("| `")] + self.assertLessEqual(len(rows), generate_site.FILES_TO_SOURCE_PART, part.name) + for row in rows: + self.assertIn(f"[Have it? Upload]({generate_site.UPLOAD_URL}", row) + + def test_the_upload_lands_in_the_console_folder(self): + with mock.patch("siterender._own_section", return_value=("Sony", "PlayStation")): + self.assertEqual(generate_site._upload_folder({"systems": ["sony-playstation"]}), + "Sony/PlayStation") + + +if __name__ == "__main__": + unittest.main()