"""Reference pages and data (round 4, package 8). The lookup answered a SHA1 without the setups that list the same file by MD5, the published folder check failed 134 EmuDeck entries a folder EmuDeck accepts, the manifest table printed `<ref>` behind a dead link and showed 17 of 48 fields, 28 system ids held files and no page, and source links went to 404s on the deploy, which has no tree cache. Tests that read the built site skip when it has not been generated. """ from __future__ import annotations import hashlib import json import re import sys import tempfile import unittest import urllib.parse from pathlib import Path from unittest import mock REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) import check_platform_folder # noqa: E402 import generate_site # noqa: E402 import release_record # noqa: E402 import sitesources # noqa: E402 import upstream # noqa: E402 PUBLIC = REPO_ROOT / "website" / "public" API = PUBLIC / "api" / "v1" DOCS = REPO_ROOT / "website" / "src" / "content" / "docs" DIST = REPO_ROOT / "website" / "dist" TMP = REPO_ROOT / "tmp" def _built(path: Path) -> Path: if not path.exists(): raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated") return path def _page(stem: str) -> str: for suffix in (".md", ".mdx"): path = DOCS / f"{stem}{suffix}" if path.is_file(): return path.read_text(encoding="utf-8") raise unittest.SkipTest(f"{stem} not generated") class LookupRecordsUnderEveryHash(unittest.TestCase): """8.1: a platform listing a file by MD5 is found by its SHA1 too.""" SHA1 = "a" * 40 FILE = {"name": "bios.bin", "path": "bios/Sony/PlayStation/bios.bin", "sha1": SHA1, "md5": "b" * 32, "crc32": "c" * 8, "sha256": "d" * 64} def records(self) -> list[dict]: db = {"files": {self.SHA1: self.FILE}, "indexes": { "by_md5": {"b" * 32: self.SHA1}, "by_crc32": {"c" * 8: self.SHA1}, "by_sha256": {"d" * 64: self.SHA1}}} coverages = {"setup": {"config": {"systems": {"sony-playstation": {"files": [ {"name": "bios.bin", "md5": "b" * 32}]}}}}} profiles = {"core": {"type": "libretro", "systems": ["sony-playstation"], "files": [{"name": "bios.bin", "crc32": "c" * 8}]}} with mock.patch.object(generate_site, "_console_system_names", return_value={}), \ mock.patch.object(generate_site, "_lookup_system_name", return_value=None), \ mock.patch.object(generate_site, "emulator_card_page", side_effect=lambda k, a: k): return generate_site.file_lookup_records( db, coverages, profiles, [{"sha1": self.SHA1, "url": "/x"}]) def test_each_hash_names_the_setup_and_the_emulator(self): records = self.records() for value in (self.SHA1, "b" * 32, "c" * 8, "d" * 64): owners = {(r["layer"], r.get("owner")) for r in records if r["value"] == value} self.assertIn(("platform", "setup"), owners, value) self.assertIn(("emulator", "core"), owners, value) def test_records_carry_the_sha1_and_say_what_was_declared(self): for record in self.records(): if record["layer"] in ("platform", "emulator"): self.assertEqual(record["sha1"], self.SHA1) copy = next(r for r in self.records() if r["layer"] == "platform" and r["value"] == self.SHA1) self.assertEqual(copy["declared"], "md5") def test_scph5501_gives_one_answer_for_its_four_hashes(self): db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) sha1 = "0555c6fae8906f3f09baf5988f00e55f88e9f30b" entry = db["files"][sha1] _built(API / "lookup") answers = [] for algorithm in ("sha1", "md5", "crc32", "sha256"): value = str(entry[algorithm]).lower() shard = json.loads((API / "lookup" / f"{value[:2]}.json").read_text()) found = [r for r in shard["items"] if r["value"] == value] answers.append(tuple(sorted({(r["layer"], r.get("owner", "")) for r in found}))) self.assertEqual(len(set(answers)), 1, answers) setups = {owner for layer, owner in answers[0] if layer == "platform"} emulators = {owner for layer, owner in answers[0] if layer == "emulator"} self.assertGreaterEqual(len(setups), 10) self.assertGreaterEqual(len(emulators), 8) class TheFolderCheckReadsEmuDecksWhitelist(unittest.TestCase): """8.2: an entry with no destination names no file. EmuDeck's check passes the system when any top-level file of the folder has its md5.""" def check(self, files: dict[str, bytes]) -> list[tuple[str, str]]: good = b"ps1 bios bytes" contract = {"verification_mode": "md5", "systems": {"sony-playstation": {"files": [ {"name": f"sony-playstation:{hashlib.md5(good).hexdigest()}", "destination": "", "md5": hashlib.md5(good).hexdigest()}, {"name": "sony-playstation:" + "0" * 32, "destination": "", "md5": "0" * 32}, ]}}} with tempfile.TemporaryDirectory(dir=TMP) as tmp: root = Path(tmp) / "bios" (root / "sub").mkdir(parents=True) for name, data in files.items(): (root / name).write_bytes(data) api = Path(tmp) / "api" api.mkdir() (api / "emudeck.json").write_text(json.dumps({"items": [{"contract": contract}]})) found = check_platform_folder.check("emudeck", root, api.as_uri() + "/") # The verifier's verdict on the matching entry, file by file. from verify import verify_entry_md5 # noqa: PLC0415 verdicts = {name: verify_entry_md5(contract["systems"]["sony-playstation"]["files"][0], str(root / name))["status"] for name in files if (root / name).is_file()} return found, verdicts def test_any_name_passes(self): found, verdicts = self.check({"my-ps1-dump.bin": b"ps1 bios bytes"}) self.assertEqual(found, [("sony-playstation: any file", "ok")]) self.assertEqual(verdicts, {"my-ps1-dump.bin": "ok"}) def test_a_folder_without_it_is_missing(self): found, _verdicts = self.check({"other.bin": b"x", "sub/deep.bin": b"ps1 bios bytes"}) self.assertEqual(found, [("sony-playstation: any file", "missing")]) def test_the_data_page_states_the_rule(self): page = generate_site.generate_data_page([]) self.assertIn("An entry with no `destination` (EmuDeck) names no file", page) self.assertLess(len((REPO_ROOT / "scripts" / "check_platform_folder.py") .read_text().splitlines()), 70) class ManifestFieldsAreComplete(unittest.TestCase): """8.3 and 8.4.""" SCHEMA = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()) def test_a_url_template_is_code(self): rows = generate_site._schema_field_rows(self.SCHEMA["$defs"]["file"], ("repo_path",)) self.assertIn("`https://raw.githubusercontent.com/Abdess/retrobios//`", rows[0]) rows = generate_site._schema_field_rows( {"properties": {"a": {"description": "From https://x.test/, then."}}}, ("a",)) self.assertEqual(rows, ["| `a` | From `https://x.test/`, then. |"]) def test_every_field_is_described_and_shown(self): page = generate_site.generate_data_page([]) for schema in (self.SCHEMA, self.SCHEMA["$defs"]["file"]): for key, spec in schema["properties"].items(): self.assertTrue(spec.get("description"), key) self.assertIn(f"| `{key}` |", page) def test_no_link_carries_an_escaped_bracket(self): html = _built(DIST / "data" / "index.html").read_text(encoding="utf-8") self.assertNotRegex(html, r'href="[^"]*&lt;') self.assertNotRegex(html, r'href="[^"]*<') class EverySystemIdWithFilesHasAPage(unittest.TestCase): """8.5: tyrian, scummvm, dos and 25 more had files and a null url.""" def test_built_index(self): items = json.loads(_built(API / "systems.json").read_text())["items"] orphans = [item["id"] for item in items if item["sha1"] and not item["url"]] self.assertEqual(orphans, []) def test_first_file_then_emulator(self): files = {"f" * 40: {"path": "bios/DOS/DOS/a.bin"}} with mock.patch("siterender._card_pages", {}): place = generate_site._first_file_place( "dos", ["f" * 40], files, {}, lambda path, sha1: "dos") self.assertEqual(place, ("DOS", "/retrobios/systems/dos/#sha1-ffffffffffff")) place = generate_site._first_file_place( "tyrian", [], files, {"opentyrian": {"systems": ["tyrian"]}}, None) self.assertEqual(place, ("", "/retrobios/emulators/opentyrian/")) # A source permalink of each forge family the pages write. _PERMALINK = re.compile( r"\]\(https://(?P[^/]+)/(?P[^/]+/[^/]+)/" r"(?:blob/|-/blob/|src/commit/)(?P[0-9a-f]{40})/(?P[^#)]+)") class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): """8.6: the deploy reads the committed record, never a tree cache.""" def test_an_address_in_a_binary_reads_as_one(self): profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} text = sitesources._source_ref_markdown( profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac") self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, " "offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`") def test_only_a_recorded_path_links(self): profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40} repo = upstream.parse_repo("https://github.com/o/r") record = {sitesources.tree_key(repo, "e" * 40): frozenset({"src/a.c"})} with mock.patch.object(sitesources, "_verified", return_value=record): sitesources._tree_paths.cache_clear() try: text = sitesources._source_ref_markdown(profile, "src/a.c:5, src/b.c:7") finally: sitesources._tree_paths.cache_clear() self.assertIn("[`src/a.c:5`](https://github.com/o/r/blob/" + "e" * 40 + "/src/a.c#L5)", text) self.assertIn("`src/b.c:7`", text) self.assertNotIn("src/b.c#", text) def test_the_writer_keeps_held_paths_only(self): profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40, "notes": "see src/c.c:3", "files": [{"source_ref": "src/a.c:5, gone.c:1"}]} with mock.patch.object(sitesources, "_cached_tree", return_value=frozenset({"src/a.c", "src/c.c", "x.c"})): held, unread = sitesources.build_source_paths({"p": profile}, [TMP]) self.assertEqual(held, {"github.com/o/r@" + "e" * 40: ["src/a.c", "src/c.c"]}) self.assertEqual(unread, []) def test_every_rendered_link_is_recorded(self): record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*")) checked, stray = 0, [] for page in pages: for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")): if "/Abdess/retrobios/" in found[0]: continue key = f"{found['host']}/{found['slug']}@{found['pin']}" path = urllib.parse.unquote(found["path"]) checked += 1 if path not in record.get(key, ()): stray.append((page.name, key, path)) self.assertGreater(checked, 1000) self.assertEqual(stray[:20], []) def test_the_record_is_read_by_the_deploy(self): """The record is committed where the build reads it, sorted.""" record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) self.assertGreater(len(record), 300) for key, paths in record.items(): self.assertRegex(key, r"^[a-z0-9.-]+/.+@[0-9a-f]{40}$") self.assertEqual(paths, sorted(set(paths)), key) class TheReferencePagesAnswerTheirWords(unittest.TestCase): """8.7, 8.8, 8.10 and 8.13.""" def test_find_or_check_a_file(self): page = generate_site.generate_files_page() self.assertTrue(page.startswith( "---\nterms: check my dump, md5 check, verify bios file, check hash\n---\n")) self.assertIn("\n# Find or check a file\n\nCheck a BIOS dump: paste its hash " "or choose the file.\n", page) def test_the_data_page(self): page = generate_site.generate_data_page([]) self.assertTrue(page.startswith( "---\nterms: manifest, schema, check a folder, rate limit, api\n---\n")) self.assertIn("## Hosting, rate limits and caching", page) self.assertIn("HTTP 429", page) self.assertIn("crypto.subtle.digest(\"SHA-1\"", page) self.assertIn("api/v1/lookup/", page) self.assertIn("`preferred: true`", page) self.assertIn("`database.json`, `stats.json`, `release.json` and the manifests are not " "envelopes", page) self.assertIn("- MinUI, MyMinUI and NextUI have no list of their own " "(`native_list: none`). An empty list does not mean nothing is needed.", page) def test_overview_ids_and_counts_are_not_indexed(self): html = _built(DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8") self.assertRegex(html, r"sony-playstation") self.assertRegex(html, r"\d+/\d+") class TheReleaseRecordHasAVersion(unittest.TestCase): """8.9.""" def test_the_writer_and_the_committed_record(self): self.assertEqual(release_record.SCHEMA_VERSION, 1) source = (REPO_ROOT / "scripts" / "release_record.py").read_text(encoding="utf-8") self.assertIn('{"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}', source) schema = json.loads((REPO_ROOT / "schemas" / "release.schema.json").read_text()) self.assertIn("schema_version", schema["required"]) record = json.loads((REPO_ROOT / "release.json").read_text()) self.assertEqual(record["schema_version"], 1) class NoEmptyCoreTable(unittest.TestCase): """8.11: MiSTer and BizHawk declare cores: [] and had a page of nothing.""" def test_own_list_platforms_get_a_sentence(self): coverages = {"misterfpga": {"platform": "MiSTer FPGA", "config": {"cores": [], "systems": {}}}} moved: dict[str, str] = {} page = generate_site.generate_cross_reference(coverages, {}, moved) self.assertEqual(moved, {}) self.assertIn(generate_site.OWN_LIST_ONLY, page) def test_built_pages(self): _built(DOCS / "cross-reference.md") for name in ("misterfpga", "bizhawk"): self.assertFalse((DOCS / f"cross-reference-{name}.md").exists()) self.assertIn(generate_site.OWN_LIST_ONLY, _page(f"platforms/{name}")) redirects = json.loads((REPO_ROOT / "website" / "src" / "generated" / "redirects.json").read_text()) self.assertEqual(redirects.get("/cross-reference-misterfpga/"), "/platforms/misterfpga/") class FilesToSourceOnAPhone(unittest.TestCase): """8.12: 601 rows on one page, no way to give a file.""" def test_parts_of_a_hundred_rows_each_with_an_upload(self): _built(DOCS / "gaps-files-to-source.md") parts = sorted(DOCS.glob("gaps-files-to-source*.md")) self.assertGreater(len(parts), 1) for part in parts: rows = [line for line in part.read_text(encoding="utf-8").splitlines() if line.startswith("| `")] self.assertLessEqual(len(rows), generate_site.FILES_TO_SOURCE_PART, part.name) for row in rows: self.assertIn(f"[Have it? Upload]({generate_site.UPLOAD_URL}", row) def test_the_upload_lands_in_the_console_folder(self): with mock.patch("siterender._own_section", return_value=("Sony", "PlayStation")): self.assertEqual(generate_site._upload_folder({"systems": ["sony-playstation"]}), "Sony/PlayStation") if __name__ == "__main__": unittest.main()