diff --git a/tests/test_site_content_data.py b/tests/test_site_content_data.py index 295ea1ba..a9c0f2d4 100644 --- a/tests/test_site_content_data.py +++ b/tests/test_site_content_data.py @@ -153,23 +153,24 @@ class NavFromFrontMatter(unittest.TestCase): def test_the_repository_nav_matches_the_pages(self): tabs = self._nav(WIKI) - self.assertEqual(tabs["Explore"][-2:], [ - {"All installer options": "wiki/installer.md"}, - {"Is it legal?": "wiki/legal.md"}, - ]) + self.assertEqual(tabs["Explore"][0], "explore.md") + self.assertEqual([next(iter(e)) for e in tabs["Explore"][1:]], ["Systems", "Emulators"]) + self.assertEqual(tabs["Your setup"][-1], {"All installer options": "wiki/installer.md"}) + start = [e if isinstance(e, str) else next(iter(e)) for e in tabs["Start here"]] + self.assertEqual(start.index("Is it legal?"), start.index("Common questions") + 1) self.assertEqual(tabs["Reference"][0], "wiki/index.md") def test_a_new_page_lands_in_its_tab_at_its_order(self): with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: wiki = _wiki_copy(tmp) (wiki / "save-files.md").write_text( - "---\nlevel: explore\norder: 35\n---\n\n# Save files\n\nText.\n", + "---\nlevel: start\norder: 45\n---\n\n# Save files\n\nText.\n", encoding="utf-8") - explore = self._nav(wiki)["Explore"] - targets = [e if isinstance(e, str) else next(iter(e.values())) for e in explore] + start = self._nav(wiki)["Start here"] + targets = [e if isinstance(e, str) else next(iter(e.values())) for e in start] self.assertEqual(targets.index("wiki/save-files.md"), - targets.index("wiki/installer.md") + 1) - self.assertIn({"Save files": "wiki/save-files.md"}, explore) + targets.index("wiki/troubleshooting.md") + 1) + self.assertIn({"Save files": "wiki/save-files.md"}, start) def test_a_page_without_a_level_fails_by_name(self): with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: @@ -200,9 +201,12 @@ class NavFromFrontMatter(unittest.TestCase): self._nav(wiki) def test_nav_false_keeps_a_page_out(self): - tabs = self._nav(WIKI) - flat = repr(tabs) - self.assertNotIn("getting-started.md", flat) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + wiki = _wiki_copy(tmp) + (wiki / "hidden.md").write_text( + "---\nlevel: start\nnav: false\n---\n\n# Hidden\n", encoding="utf-8") + flat = repr(self._nav(wiki)) + self.assertNotIn("hidden.md", flat) def test_every_wiki_page_states_its_place(self): for page in sorted(WIKI.glob("*.md")): diff --git a/tests/test_site_r2_wiki.py b/tests/test_site_r2_wiki.py new file mode 100644 index 00000000..361114bd --- /dev/null +++ b/tests/test_site_r2_wiki.py @@ -0,0 +1,372 @@ +"""Wiki copy and the diagrams module (round 2, package 5). + +Each class names the finding it holds: a safety line the code contradicted, +install commands with no Windows form, a run's end described wrongly, pages +with two names, an FAQ answer the setups' own checks contradict, a setup +count given two ways, unreadable diagram text, and example hashes that +answered a hash search in place of the file card. +""" + +from __future__ import annotations + +import json +import re +import sys +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT)) + +import sitediagrams # noqa: E402 +from siteguides import INSTALL_BASE # noqa: E402 + +from tests import sitepages # noqa: E402 + +WIKI = REPO_ROOT / "wiki" +FENCED = re.compile(r"^```.*?^```", re.S | re.M) + + +def _front(text: str) -> dict: + match = re.match(r"---\n(.*?)\n---\n", text, re.S) + return yaml.safe_load(match.group(1)) if match else {} + + +def _h1(text: str) -> str: + match = re.search(r"^# (.+?)(?:\s*\{[^}]*\})?\s*$", FENCED.sub("", text), re.M) + return match.group(1) if match else "" + + +def _section(text: str, heading: str) -> str: + start = re.search(rf"(?m)^#+ {re.escape(heading)}(?:\s*\{{[^}}]*\}})?\n", text) + if start is None: + raise AssertionError(f"no section {heading!r}") + rest = text[start.end():] + end = re.search(r"(?m)^#{1,2} ", FENCED.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), rest)) + return rest[: end.start()] if end else rest + + +def _wiki(name: str) -> str: + return (WIKI / name).read_text(encoding="utf-8") + + +def _built(name: str) -> str: + sitepages.require() + return sitepages.page(name).read_text(encoding="utf-8") + + +START_PAGES = ("what-are-bios-files.md", "first-install.md", "troubleshooting.md", + "faq.md", "glossary.md", "legal.md") + + +class SafetyListMatchesTheInstaller(unittest.TestCase): + """C1: "It writes only in your BIOS folder", while install.py writes the + entries marked root: save into the setup's save folder.""" + + def test_the_faq_names_the_save_folder_while_a_manifest_writes_there(self): + saved = [ + path.name for path in sorted((REPO_ROOT / "install").glob("*.json")) + if any(entry.get("root") == "save" + for entry in json.loads(path.read_text(encoding="utf-8")).get("files", [])) + ] + section = _section(_wiki("faq.md"), "Is the command safe?") + if saved: + self.assertNotIn("only in your BIOS folder", section) + self.assertIn("save folder", section) + self.assertIn("retrobios-backup", section) + self.assertIn("--restore", section) + + +class InstallCommandsHaveEveryForm(unittest.TestCase): + """N7, B3, C6: troubleshooting and the FAQ gave only `curl ... | sh`. + Every install command on a Start here page is marked for the system tabs, + and the built page shows the PowerShell form beside it.""" + + def test_every_install_command_on_a_start_page_is_marked(self): + for name in START_PAGES: + text = _wiki(name) + with self.subTest(page=name): + for match in re.finditer(r"(?m)^```(?:sh|bash)\n(.*?)\n```", text, re.S): + if "install.sh" not in match.group(1) or "|" not in match.group(1): + continue + before = text[: match.start()].rstrip("\n").rsplit("\n", 1)[-1] + self.assertRegex(before, r"\n```sh\n" + f"curl -fsSL {INSTALL_BASE}/install.sh \\\n | sh -s -- --platform retroarch --dest /a/b\n```\n" + ) + text = sitediagrams.expand_commands(page) + windows, rest = text.split('', 1) + self.assertIn("--dest E:\\x", windows) + self.assertNotIn("/a/b", windows) + self.assertIn("| sh -s -- --platform retroarch --dest /a/b", rest) + + def test_a_marked_block_without_an_install_command_fails_the_build(self): + with self.assertRaises(ValueError): + sitediagrams.expand_commands("\n```sh\nls -l\n```\n") + + +class FirstInstallEnds(unittest.TestCase): + """C7, B15: the page said `Done.` may list failures, and sent a wrong + folder to a section with no answer for it.""" + + def test_the_wrong_folder_answer_is_dest(self): + section = _section(_wiki("first-install.md"), "2. Check the folder it found") + self.assertIn("--dest", section) + self.assertIn("", section) + self.assertNotIn("Batocera", section) + + def test_the_diagram_follows_the_steps(self): + text = _wiki("first-install.md") + self.assertLess(text.index("## 5. Read the last line"), text.index("")) + + +class OneNamePerPage(unittest.TestCase): + """N11, C4, V22: "Common questions" in the menu opened "FAQ".""" + + def test_no_wiki_page_names_itself_twice(self): + for path in sorted(WIKI.glob("*.md")): + text = path.read_text(encoding="utf-8") + with self.subTest(page=path.name): + self.assertNotIn("nav_title", _front(text)) + + def test_links_to_a_renamed_page_use_its_heading(self): + names = {path.name: _h1(path.read_text(encoding="utf-8")) for path in WIKI.glob("*.md")} + stale = {"faq.md": "FAQ", "glossary.md": "Glossary", "legal.md": "The project's legal position"} + for path in sorted(WIKI.glob("*.md")): + for text, target in re.findall(r"\[([^\]]+)\]\(([a-z-]+\.md)\)", path.read_text(encoding="utf-8")): + if target in stale and target in names: + with self.subTest(page=path.name, link=text): + self.assertNotEqual(text, stale[target]) + + +class ArcadeFilesDependOnTheSetup(unittest.TestCase): + """C11, C02: "Does a BIOS work on every setup? Yes." Platforms check an + MD5 and refuse another revision, and neogeo.zip differs by setup.""" + + def test_the_answer_is_not_a_plain_yes(self): + section = _section(_wiki("faq.md"), "Does a BIOS work on every setup?").strip() + self.assertFalse(section.startswith("Yes")) + self.assertIn("neogeo.zip", section) + + def test_the_setups_do_install_different_neogeo_zips(self): + digests = set() + for path in (REPO_ROOT / "install").glob("*.json"): + for entry in json.loads(path.read_text(encoding="utf-8")).get("files", []): + if str(entry.get("dest", "")).lower().endswith("/neogeo.zip") or entry.get("dest") == "neogeo.zip": + digests.add(entry.get("sha1")) + self.assertGreater(len(digests), 1) + + def test_zip_or_unzip_is_answered(self): + section = _section(_wiki("faq.md"), "Zip or unzip?") + self.assertIn("neogeo.zip", section) + + +class VerificationModesSplit(unittest.TestCase): + """C12, N24, F5: the page opened on "Before a game starts", ran 37 + entries of contents, and cited checker code as plain text.""" + + def test_opening(self): + text = _wiki("verification-modes.md") + self.assertNotIn("Before a game starts", text) + self.assertIn("Most setups can show which BIOS files are missing.", text) + + def test_the_project_reports_have_their_own_page(self): + modes = _wiki("verification-modes.md") + reports = _wiki("reading-reports.md") + for heading in ("## Hash mismatch / UNTESTED", "## verify.py reports errors", "### Reading the output"): + with self.subTest(heading=heading): + self.assertNotIn(heading, modes) + self.assertIn(heading, reports) + self.assertEqual(_front(reports).get("level"), "reference") + + def test_checker_links_are_pinned_to_the_revision_the_platform_names(self): + named = " ".join( + f"{data.get('version', '')} {data.get('source', '')}" + for data in (yaml.safe_load(p.read_text(encoding="utf-8")) or {} + for p in (REPO_ROOT / "platforms").glob("*.yml")) + if isinstance(data, dict) + ) + links = re.findall(r"\((https://(?:github\.com|gitlab\.com)/[^)\s]+)\)", _wiki("verification-modes.md")) + pinned = [url for url in links if re.search(r"/-?/?blob/", url)] + self.assertGreaterEqual(len(pinned), 10) + for url in pinned: + revision = re.search(r"/blob/([^/]+)/", url).group(1) + with self.subTest(url=url): + self.assertNotIn(revision, ("main", "master", "HEAD")) + self.assertIn(revision, named) + + +class SetupCount(unittest.TestCase): + """C2: the home page said 15 more, the diagram 14 more, from two counts.""" + + def test_one_count(self): + from common import list_registered_platforms + + self.assertEqual(len(sitediagrams.setups()), + len(list_registered_platforms(str(REPO_ROOT / "platforms"), include_archived=True))) + + def test_every_rendered_count_agrees(self): + """Each "RetroArch, Batocera, EmuDeck and N more" names three setups.""" + found = set() + for path in [*sitepages.pages()]: + text = path.read_text(encoding="utf-8") + found.update(int(n) for n in re.findall(r"(?:RetroArch|EmuDeck|Batocera) and (\d+) more", text)) + found.update(int(n) for n in re.findall( + r"and (\d+) more", sitediagrams.og_card_svg("home", b"png"))) + self.assertTrue(found) + self.assertEqual(found, {len(sitediagrams.setups()) - 3}) + + +class Vocabulary(unittest.TestCase): + """C5: one thing called device, setup, frontend, platform and game menu.""" + + def test_the_glossary_defines_the_words_the_reference_uses(self): + terms = {term for term in re.findall(r"^\*\*([^*]+)\*\*", _wiki("glossary.md"), re.M)} + terms |= set(re.findall(r"\*\*([^*]+)\*\*", _wiki("glossary.md"))) + for word in ("Pack", "Platform", "MD5", "SHA-1", "Setup"): + with self.subTest(word=word): + self.assertIn(word, terms) + + def test_start_pages_and_the_first_diagram_say_setup(self): + for name in START_PAGES[:5]: + text = FENCED.sub("", _wiki(name)).lower() + with self.subTest(page=name): + self.assertNotRegex(text, r"\bfrontends?\b") + self.assertNotIn("game menu", text) + for figure in ("bios-chain", "install-flow"): + with self.subTest(figure=figure): + self.assertNotIn("rontend", sitediagrams.figure_html(figure)) + + +class NavigationPlaces(unittest.TestCase): + """N14, N15, N16, T5: installer and legal in Explore, build internals in + Reference, and a legacy Getting started page in no tab.""" + + def test_front_matter(self): + expected = { + "installer.md": "setup", "legal.md": "start", + "architecture.md": "contribute", "tools.md": "contribute", + "advanced-usage.md": "contribute", "reference-terms.md": "contribute", + } + for name, level in expected.items(): + with self.subTest(page=name): + self.assertEqual(_front(_wiki(name))["level"], level) + self.assertEqual(_front(_wiki("legal.md"))["order"], _front(_wiki("faq.md"))["order"] + 5) + self.assertEqual(_front(_wiki("verification-modes.md"))["group"], "Checking files") + self.assertEqual(_front(_wiki("community-tools.md"))["group"], "Data") + self.assertFalse((WIKI / "getting-started.md").exists()) + + def test_no_next_aside_is_left(self): + for path in sorted(WIKI.glob("*.md")): + with self.subTest(page=path.name): + self.assertNotIn(":::tip[Next]", path.read_text(encoding="utf-8")) + + +class TroubleshootingLines(unittest.TestCase): + """C22: "Details: When something goes wrong", a sudo line that is the + installer's to give, and an unclear Linux line.""" + + def test_lines(self): + text = _wiki("troubleshooting.md") + self.assertNotIn("Details: [When something goes wrong]", text) + self.assertIn("Other messages: [When something goes wrong]", text) + self.assertIn("Run the command once.", text) + for name in ("troubleshooting.md", "first-install.md"): + with self.subTest(page=name): + self.assertNotIn("sudo", _wiki(name)) + self.assertRegex(" ".join(_wiki(name).split()), r"[Dd]o what the message says") + + def test_the_installer_names_what_to_run(self): + source = (REPO_ROOT / "install.py").read_text(encoding="utf-8") + advice = source[source.index("def _write_advice"):source.index("def _plural")] + self.assertIn("_INVOCATION.command(plat, path, elevated=True)", advice) + + +class DiagramText(unittest.TestCase): + """V17: 11 px text in fixed-height boxes, paths broken mid-token.""" + + def test_text_reads_at_14_px_in_the_column(self): + self.assertGreaterEqual(sitediagrams.TEXT_SIZE * sitediagrams.COLUMN / sitediagrams.WIDE, 14) + + def test_no_box_holds_more_than_three_lines(self): + wide = sitediagrams.WIDE - 2 + layouts = { # the box width of each wide layout: flow, grid of two, columns + "bios-chain": (sitediagrams._bios_chain_boxes(), (wide - 26 * 3) / 4), + "install-flow": (sitediagrams._install_boxes(), (wide - 18) / 2), + "verification-modes": (sitediagrams._mode_boxes(), (wide - 16 * 2) / 3), + } + for name, (boxes, box_width) in layouts.items(): + for width in (sitediagrams.NARROW - 2, box_width): + for box in boxes: + with self.subTest(figure=name, box=box.title, width=width): + self.assertLessEqual(len(sitediagrams.box_lines(box, width)[1]), + sitediagrams.MAX_BOX_LINES) + + def test_a_path_is_mono_on_its_own_line(self): + html = sitediagrams.figure_html("bios-chain") + self.assertRegex(html, r']*style="font-family: var\(--sl-font-mono[^>]*>/userdata/bios/') + + def test_boxes_take_the_height_of_their_lines(self): + width, _height, body = sitediagrams.flow(sitediagrams._bios_chain_boxes(), False, sitediagrams.WIDE) + heights = {float(h) for h in re.findall(r']*height="([\d.]+)"[^>]*class="rb-d-box', "".join(body))} + self.assertGreater(len(heights), 1) + + +class ExampleHashesStayOutOfSearch(unittest.TestCase): + """F4: the scph5501 MD5 typed in search found four documentation pages + and not the file card.""" + + def test_wiki_hashes_are_excluded(self): + text = _built("wiki/verification-modes.md") + for match in re.finditer(r"\b[0-9a-f]{32}\b", text): + line_start = text.rfind("\n", 0, match.start()) + context = text[line_start: match.end() + 10] + with self.subTest(hash=match.group(0)): + self.assertIn("data-pagefind-ignore", context) + + def test_a_fenced_hash_is_wrapped(self): + out = sitediagrams.hide_hashes("```sh\nx 490f666e1afb15b7362b406ed1cea246\n```\n") + self.assertTrue(out.startswith("
\n\n```sh")) + + def test_only_wiki_pages_are_touched(self): + page = "Hash `490f666e1afb15b7362b406ed1cea246`.\n" + self.assertEqual(sitediagrams.prepare_page(page, Path("systems/sony.md")), page) + self.assertIn("data-pagefind-ignore", sitediagrams.prepare_page(page, Path("wiki/x.md"))) + + +class InstallerOptionsAtAGlance(unittest.TestCase): + """C23: a 9,000-word page with no table at the top. The table is read + from the parser, so it lists what --help lists.""" + + def test_every_option_of_the_parser_has_a_row(self): + source = (REPO_ROOT / "install.py").read_text(encoding="utf-8") + declared = set(re.findall(r'parser\.add_argument\(\s*"(--[a-z-]+)"', source)) + listed = {usage.split()[0].rstrip(",") for usage, _ in sitediagrams.installer_options()} + self.assertEqual(declared | {"--help"}, listed) + + def test_the_built_page_opens_on_the_table(self): + text = _built("wiki/installer.md") + table = text.index("| Option | What it does | More |") + self.assertLess(table, text.index("## One line")) + self.assertIn("| `--check` |", text) + self.assertNotIn(" + ## One line ```bash @@ -810,7 +817,7 @@ what it must hash to. Entries the collection cannot satisfy are not silently dropped: they travel in `omitted_files` with a reason, and the run prints the count before downloading anything. `not_found` means no file in the collection matches the entry, which -is what the [gap analysis](../gaps.md) tracks. +is what the gap analysis tracks (Reference: [What the packs do not hold](../gaps.md)). Some cores read a whole tree of data rather than a few files: Dolphin's `Sys/`, PPSSPP's assets, blueMSX's machines. These are not files of the collection, so @@ -950,7 +957,7 @@ no event at all. A program reads a missing `end` as that case. | Variable | Effect | |----------|--------| -| `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags ([release process](release-process.md#cutting-a-release)), so a tag holds a file list until the next release | +| `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags, so a tag holds a file list until the next release. For contributors: [release process](release-process.md#cutting-a-release) | | `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check. A file list that cannot be fetched names the address, and this variable when it is set | | `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`, `android`) instead of detecting it | | `EXTERNAL_STORAGE` | Android only, the shared storage root RetroArch builds its default directories from. Defaults to `/storage/emulated/0` | diff --git a/wiki/profiling.md b/wiki/profiling.md index 7b976038..d37add25 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -1,7 +1,6 @@ --- level: contribute order: 10 -nav_title: Profiling an emulator --- # Profiling guide diff --git a/wiki/reading-reports.md b/wiki/reading-reports.md new file mode 100644 index 00000000..8f202002 --- /dev/null +++ b/wiki/reading-reports.md @@ -0,0 +1,152 @@ +--- +level: reference +order: 15 +group: Checking files +--- + +# Reading this project's own reports + +These sections are about this repository's own tools, run from a clone. +`scripts/verify.py` checks the collection in `bios/` against each platform, never +your BIOS folder: for that, use the installer's `--check` +([When a game does not start](troubleshooting.md)). How each platform checks a +file is on [How each platform checks files](verification-modes.md). + +## Hash mismatch / UNTESTED + +`verify.py` reports `UNTESTED` for a file of the collection that was hashed +and does not match what the platform declares. The line gives the reason: + +``` + UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file +``` + +The system page on the site lists every known version of the file with its +expected MD5 and SHA1. + +### Existence-mode platforms never report UNTESTED + +On RetroArch, Lakka and RetroPie, the platform check only asks whether the file +is there, so the only outcomes are OK and MISSING. `verify.py` adds the emulator +ground truth from the profiles and flags a `DISCREPANCY` when a file passes the +platform check but fails the emulator's own size or hash validation. + +### The .variants/ directory + +When multiple versions of the same BIOS exist (different revisions, regions, or +dumps), the primary version lives in the main directory and alternatives live in +`.variants/`. `verify.py` checks the primary file first, then falls back to +variants when resolving by hash. + +A file that matches a variant hash but not the primary works. It is not the +version the emulator tries first, and some cores accept several versions. + +## Pack is missing files + +A generated pack doesn't contain all the files you expected. + +### Severity levels + +`verify.py` assigns a severity to each issue, counted under `severity_counts` +in its `--json` output. Not all missing files are equally important: + +| Severity | Meaning | Action needed | +|----------|---------|---------------| +| CRITICAL | Required file missing on an MD5 or SHA1 platform | Must fix. Core won't function. | +| WARNING | Optional file missing on an MD5 or SHA1 platform, hash mismatch (UNTESTED) on any hash platform, or a required file missing on an existence platform | Core works but with reduced functionality, or runs on content nobody verified. | +| INFO | Optional file missing on an existence-only platform, or a missing file the core covers with an HLE fallback | Core works fine, BIOS improves accuracy. | +| OK | File present and verified | No action needed. | + +The full mapping, including how `hle_fallback` overrides the platform mode, is +in the [severity matrix](verification-modes.md#severity-matrix). + +Focus on CRITICAL issues first. WARNING files improve the experience but aren't +strictly necessary. INFO files are nice to have. + +### Large files (release assets) + +Files like PS3UPDAT.PUP, PSVUPDAT.PUP, and Switch firmware are too large for the +git repository. They are stored as GitHub release assets under the `large-files` +release and downloaded at build time. + +If a pack build fails to include these, check your network connection. In offline +mode (`--offline`), large files are only included if already cached locally in +`.cache/large/`. + +### Data directories + +Some cores need entire directory trees rather than individual files (e.g. Dolphin's +`Sys/` directory, PPSSPP's `assets/`). These are fetched by `refresh_data_dirs.py` +from upstream repositories. + +In offline mode, data directories are only included if already cached in `data/`. +Run `python scripts/refresh_data_dirs.py` to fetch them. + +## verify.py reports errors + +How to read and interpret `verify.py` output. + +### Status codes + +| Status | Meaning | +|--------|---------| +| `OK` | File present, hash matches (or existence check passed) | +| `UNTESTED` | File present, hash not confirmed against the expected value | +| `MISSING` | File not found in the repository | +| `DISCREPANCY` | The platform check passes, an emulator's own check fails | + +### Reading the output + +The first line counts the platform's files and names its check. Under it come +the entries that need attention, one per line as `STATUS (required|optional): +system/file`, followed by a reason where there is one, then the counts of the +files the emulator cores add. From `python scripts/verify.py --platform retrodeck`: + +``` +RetroDECK: 2007/2008 OK, 1 untested [md5] + UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file + DISCREPANCY: nintendo-snes/sgb2.boot.rom: RetroDECK says OK but supersnes9x says size mismatch: got 524288, accepted [256] + supersnes9x [size] + Core files: 1495 in pack, 0 required missing, 1 optional missing +``` + +`--verbose` also lists the passing entries that an emulator validates, with +the check and the source lines behind it: + +``` +Recalbox: 351/351 OK [md5] + OK (required): macintosh/MacII.ROM + minivmac validates size=262144 [minivmac/cfg/CNFGRAPI.h:32, minivmac/src/OSGLUERETRO.c:781] +``` + +### Cross-reference section + +After per-file results, `verify.py` prints a cross-reference report. This lists +files that emulator cores need but that the platform YAML doesn't declare. These +files are still included in packs automatically, but the report helps identify +gaps in platform coverage data. + +The cross-reference uses `resolve_platform_cores()` to determine which emulator +profiles are relevant for each platform, then checks whether each profile's files +appear in the platform config. + +### Filtering output + +```bash +# By platform +python scripts/verify.py --platform batocera + +# By emulator core +python scripts/verify.py --emulator beetle_psx + +# By system +python scripts/verify.py --system sony-playstation + +# By hardware target +python scripts/verify.py --platform retroarch --target switch + +# JSON for scripted processing: the entries that need attention +python scripts/verify.py --platform retroarch --json +# every entry, passing ones included, with its ground truth +python scripts/verify.py --platform retroarch --json --verbose +``` diff --git a/wiki/reference-terms.md b/wiki/reference-terms.md index 9ed2edfe..7f892478 100644 --- a/wiki/reference-terms.md +++ b/wiki/reference-terms.md @@ -1,7 +1,6 @@ --- -level: reference -order: 130 -nav_title: Project terms +level: contribute +order: 90 --- # Terms of the reference pages @@ -149,8 +148,8 @@ everyday words (BIOS, emulator, core, hash) are in the [glossary](glossary.md). takes a file from (the collection, a data folder, a release asset) is its source, not its provenance. -:::tip[Next] +## Next steps + - [How each platform checks files](verification-modes.md) -- [Glossary](glossary.md) -::: +- [Words used on this site](glossary.md) diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 353e260c..6fccb7db 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -1,7 +1,6 @@ --- level: contribute order: 40 -nav_title: Testing --- # Testing guide @@ -56,7 +55,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture @@ -398,11 +397,12 @@ nav_title: Save files # only when the menu names it otherwise than its headin two without renumbering. The generated pages (systems, emulators, the gaps and data pages) hold fixed places among them, set in `generate_nav`. - `nav_title` is the menu label. Without it the menu shows the page's `# ` - heading. It is not `title`, which both MkDocs and Astro Starlight read as - the page's own title. + heading, which is the rule: a page keeps one name in the menu, its heading + and the links to it. It is not `title`, which both MkDocs and Astro + Starlight read as the page's own title. +- `group` places a Reference page under a group of that tab, such as + `Checking files` or `Data`. - `nav: false` keeps a page out of the menu on purpose. - `getting-started.md` does, since it only links the pages that took its - content. `python scripts/generate_site.py` stops with the page's name when `level` or `order` is missing, or when two pages of a tab take the same `order`. diff --git a/wiki/tools.md b/wiki/tools.md index c84eabb8..aabf6b41 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1,6 +1,6 @@ --- -level: reference -order: 110 +level: contribute +order: 70 --- # Command-line tools diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 6d7542d3..0c7df1ab 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -1,17 +1,18 @@ --- level: reference order: 10 +group: Checking files --- # How each platform checks files -Before a game starts, your setup may check its BIOS files, and each setup checks -differently. RetroArch only asks whether a file with the right name is in the -folder. Batocera computes the file's MD5 fingerprint and compares it with its -list, so a renamed or wrong-region file shows as `UNTESTED`: present, but not -the expected file. BizHawk does the same with SHA1. The way a setup checks -decides what "OK" means there: a file RetroArch accepts can still be one -Batocera reports. +Most setups can show which BIOS files are missing. Each one checks differently. +RetroArch only asks whether a file with the right name is in the folder. +Batocera computes the file's MD5 and compares it with its list, so a +wrong-region file, or another file given the right name, shows as `UNTESTED`: +present, but not the expected file. BizHawk does the same with SHA1. A file +RetroArch accepts can still be one Batocera reports. The emulator, not the +check, is what fails when a game starts. Example: `scph5501.bin`, MD5 `490f666e1afb15b7362b406ed1cea246`. Any file named `scph5501.bin` passes on RetroArch. On Batocera only those exact bytes pass. @@ -24,11 +25,14 @@ changes ([severity matrix](#severity-matrix)). +Each reference to a checker's code below links the revision of that code the +platform's own list was read from. + ## Existence mode **Platforms**: RetroArch, Lakka, RetroPie, MinUI, NextUI, MyMinUI -**Source**: RetroArch `core_info.c`, function `path_is_valid()`; minarch for the MinUI family +**Source**: RetroArch v1.22.2 [`core_info.c:2285`](https://github.com/libretro/RetroArch/blob/v1.22.2/core_info.c#L2285), `path_is_valid()`. For the MinUI family, minarch. A file is OK if it exists at the expected path. No hash is checked. Any file with the correct name passes, regardless of content. @@ -47,7 +51,7 @@ Lakka and RetroPie inherit this behavior through platform config inheritance MinUI, NextUI and MyMinUI check nothing at all. They hand each core its own folder, `Bios//`, and open nothing in it: minarch answers -`RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY` with that folder (MinUI `workspace/all/minarch/minarch.c:1889-1893` and `:2943`). The +`RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY` with that folder (MinUI [`workspace/all/minarch/minarch.c:1889-1893`](https://github.com/shauninman/MinUI/blob/dbf89435a5179788418ff3573042cf9c7e75d9e0/workspace/all/minarch/minarch.c#L1889-L1893) and [`:2943`](https://github.com/shauninman/MinUI/blob/dbf89435a5179788418ff3573042cf9c7e75d9e0/workspace/all/minarch/minarch.c#L2943)). The core decides what it loads, so the files are the ones its profile documents, placed below each folder a pak hands it (`core_system_dirs` in the platform YAML). @@ -62,6 +66,9 @@ The details vary by platform. ### Standard MD5 (Batocera, RetroBat) +**Source**: Batocera 43.1 `batocera-systems`, [`md5sum()`](https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/package/batocera/core/batocera-scripts/scripts/batocera-systems#L987-L992) and +function [`checkBios`](https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/package/batocera/core/batocera-scripts/scripts/batocera-systems#L1047). + `verify.py` replicates Batocera's `md5sum()` function. The file is read in binary mode, hashed with MD5, and compared case-insensitively against the expected value. @@ -77,7 +84,7 @@ and returns OK directly. ### MD5 shorter than 32 characters (Batocera) -Batocera's script compares the full string (`batocera-systems:1088`), so an MD5 +Batocera's script compares the full string ([`batocera-systems:1088`](https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/package/batocera/core/batocera-scripts/scripts/batocera-systems#L1088)), so an MD5 shorter than 32 characters matches no file there. The Batocera scraper completes such a value when exactly one MD5 in `database.json` starts with it (`_resolve_truncated_md5` in `scripts/scraper/batocera_scraper.py`) and keeps it as @@ -86,11 +93,12 @@ characters by prefix, and only for a file whose name the entry also names. ### md5_composite (Recalbox ZIP verification) -Recalbox hashes a ZIP with `Zip::Md5Composite` (`es-core/src/utils/Zip.cpp:71-96` in -recalbox-emulationstation at `f4cd0ae4`): one MD5 over the contents of every member, -read in filename order. The members' own MD5s are not involved, and neither is the -MD5 of the ZIP file itself: `Bios::Scan` takes the composite for every `.zip` -(`es-app/src/bios/Bios.cpp:64-68`). `verify.py` computes the same value with +Recalbox hashes a ZIP with `Zip::Md5Composite` +([`es-core/src/utils/Zip.cpp:71-96`](https://gitlab.com/recalbox/recalbox/-/blob/10.1.1/projects/frontend/es-core/src/utils/Zip.cpp#L71-96) in +Recalbox 10.1.1): one MD5 over the contents of every member, read in filename +order. The members' own MD5s are not involved, and neither is the MD5 of the ZIP +file itself: `Bios::Scan` takes the composite for every `.zip` +([`es-app/src/bios/Bios.cpp:86-91`](https://gitlab.com/recalbox/recalbox/-/blob/10.1.1/projects/frontend/es-app/src/bios/Bios.cpp#L86-91)). `verify.py` computes the same value with `md5_composite()` (`scripts/hashing.py`). The registry gives Recalbox `zip_digest: composite`. For a platform so marked, a @@ -109,8 +117,9 @@ listed hash is OK. Recalbox colours each entry of `es_bios.xml` from two attributes, `mandatory` and `hashMatchMandatory`, both true unless written `"false"` -(`es-app/src/bios/Bios.cpp:162-163` in recalbox-emulationstation at `f4cd0ae4`, -colours set by `Bios::Scan` at lines 84-108). The platform YAML carries them as +([`es-app/src/bios/Bios.cpp:187-188`](https://gitlab.com/recalbox/recalbox/-/blob/10.1.1/projects/frontend/es-app/src/bios/Bios.cpp#L187-188) in +Recalbox 10.1.1, colours set by `Bios::Scan` at +[lines 106-130](https://gitlab.com/recalbox/recalbox/-/blob/10.1.1/projects/frontend/es-app/src/bios/Bios.cpp#L106-130)). The platform YAML carries them as `required` and, when the attribute is written, `hash_match_mandatory`. | File | mandatory | hashMatchMandatory | Recalbox | verify.py | @@ -130,7 +139,7 @@ Recalbox shows RED for a mandatory entry with `hashMatchMandatory`. When a platform entry has a `zipped_file` field, the expected MD5 is not the hash of the ZIP container but of a specific ROM file inside the ZIP. `verify.py` replicates Batocera's -`checkInsideZip()`: +[`checkInsideZip()`](https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/package/batocera/core/batocera-scripts/scripts/batocera-systems#L994): 1. Open the ZIP. 2. Find the inner file by name (case-insensitive via `casefold()`). @@ -141,7 +150,7 @@ If the inner file is not found inside the ZIP, the status is UNTESTED with a rea ### RomM verification -**Source**: `backend/models/firmware.py`, `Firmware.verify_file_hashes` +**Source**: RomM 5.4.0 [`backend/models/firmware.py:58-76`](https://github.com/rommapp/romm/blob/5.4.0/backend/models/firmware.py#L58-L76), `Firmware.verify_file_hashes` RomM reads the file's bytes (`verification_mode: md5`) but does not compare one digest. Its platform YAML carries `hash_rule: size_and_any_hash`: the file's @@ -157,7 +166,7 @@ not a single digest gets a new `hash_rule` value, never a new mode. ### ROCKNIX verification -**Source**: `rocknix-systems`, function `checkBios` +**Source**: ROCKNIX 20261001 [`rocknix-systems:137-197`](https://github.com/ROCKNIX/distribution/blob/20261001/projects/ROCKNIX/packages/rocknix/sources/scripts/rocknix-systems#L137-L197), function `checkBios` ROCKNIX derives its BIOS checker from Batocera's, so the logic has the same shape: `md5sum()` on the file, `checkInsideZip()` for archive entries, and an @@ -167,8 +176,9 @@ platform declares is mandatory in its own data, so the platform YAML marks all ### KNULLI, AmberELEC and EmuELEC verification -**Source**: `knulli-systems` (KNULLI) and `batocera-systems` (AmberELEC, -EmuELEC), function `checkBios` +**Source**: function `checkBios` in [`knulli-systems:867-928`](https://github.com/knulli-cfw/knulli-linux/blob/20260511/package/system/knulli-scripts/scripts/knulli-systems#L867-L928) +(KNULLI 20260511), and in `batocera-systems` for +[AmberELEC](https://github.com/AmberELEC/AmberELEC/blob/a21975bfb1a3af68f66e07a931b994b523e1e6e2/packages/amberelec/sources/scripts/batocera-systems#L583-L644) and [EmuELEC 4.8](https://github.com/EmuELEC/EmuELEC/blob/v4.8/packages/sx05re/emuelec/bin/batocera/batocera-systems#L817-L834) All three keep Batocera's `systems` dict and its checker, run by their EmulationStation's BIOS screen. @@ -205,6 +215,8 @@ URL and install themselves, so they are not part of the pack. **Platforms**: BizHawk +**Source**: BizHawk 2.11.1 [`FirmwareDatabase.cs`](https://github.com/TASEmulators/BizHawk/blob/2.11.1/src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs) + BizHawk firmware entries use SHA1 as the primary hash. `verify.py` computes SHA1 via `compute_hashes()` and compares case-insensitively. @@ -437,152 +449,8 @@ dump with that CRC32 too (`bios/Dragon/Dragon/d64_1.rom`), but its MD5 is would not accept it at that path (step 3). The RetroBat file ships, and the [gaps page](../gaps.md) lists the discrepancy with the dump XRoar accepts. -## Reading this project's own reports +## Next steps -The sections below are about `verify.py` run on a clone of the repository. -Your own BIOS folder is checked with the installer's `--check` -([When a game does not start](troubleshooting.md)). - -## Hash mismatch / UNTESTED - -This section and the two after it are about this repository's own tools, run -from a clone. `scripts/verify.py` checks the collection in `bios/` against each -platform, never your BIOS folder: for that, use the installer's `--check` -([When a game does not start](troubleshooting.md)). - -`verify.py` reports `UNTESTED` for a file of the collection that was hashed -and does not match what the platform declares. The line gives the reason: - -``` - UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file -``` - -The system page on the site lists every known version of the file with its -expected MD5 and SHA1. - -### Existence-mode platforms never report UNTESTED - -On RetroArch, Lakka and RetroPie, the platform check only asks whether the file -is there, so the only outcomes are OK and MISSING. `verify.py` adds the emulator -ground truth from the profiles and flags a `DISCREPANCY` when a file passes the -platform check but fails the emulator's own size or hash validation. - -### The .variants/ directory - -When multiple versions of the same BIOS exist (different revisions, regions, or -dumps), the primary version lives in the main directory and alternatives live in -`.variants/`. `verify.py` checks the primary file first, then falls back to -variants when resolving by hash. - -A file that matches a variant hash but not the primary works. It is not the -version the emulator tries first, and some cores accept several versions. - -## Pack is missing files - -A generated pack doesn't contain all the files you expected. - -### Severity levels - -`verify.py` assigns a severity to each issue, counted under `severity_counts` -in its `--json` output. Not all missing files are equally important: - -| Severity | Meaning | Action needed | -|----------|---------|---------------| -| CRITICAL | Required file missing on an MD5 or SHA1 platform | Must fix. Core won't function. | -| WARNING | Optional file missing on an MD5 or SHA1 platform, hash mismatch (UNTESTED) on any hash platform, or a required file missing on an existence platform | Core works but with reduced functionality, or runs on content nobody verified. | -| INFO | Optional file missing on an existence-only platform, or a missing file the core covers with an HLE fallback | Core works fine, BIOS improves accuracy. | -| OK | File present and verified | No action needed. | - -The full mapping, including how `hle_fallback` overrides the platform mode, is -in [severity matrix](#severity-matrix). - -Focus on CRITICAL issues first. WARNING files improve the experience but aren't -strictly necessary. INFO files are nice to have. - -### Large files (release assets) - -Files like PS3UPDAT.PUP, PSVUPDAT.PUP, and Switch firmware are too large for the -git repository. They are stored as GitHub release assets under the `large-files` -release and downloaded at build time. - -If a pack build fails to include these, check your network connection. In offline -mode (`--offline`), large files are only included if already cached locally in -`.cache/large/`. - -### Data directories - -Some cores need entire directory trees rather than individual files (e.g. Dolphin's -`Sys/` directory, PPSSPP's `assets/`). These are fetched by `refresh_data_dirs.py` -from upstream repositories. - -In offline mode, data directories are only included if already cached in `data/`. -Run `python scripts/refresh_data_dirs.py` to fetch them. - -## verify.py reports errors - -How to read and interpret `verify.py` output. - -### Status codes - -| Status | Meaning | -|--------|---------| -| `OK` | File present, hash matches (or existence check passed) | -| `UNTESTED` | File present, hash not confirmed against the expected value | -| `MISSING` | File not found in the repository | -| `DISCREPANCY` | The platform check passes, an emulator's own check fails | - -### Reading the output - -The first line counts the platform's files and names its check. Under it come -the entries that need attention, one per line as `STATUS (required|optional): -system/file`, followed by a reason where there is one, then the counts of the -files the emulator cores add. From `python scripts/verify.py --platform retrodeck`: - -``` -RetroDECK: 2007/2008 OK, 1 untested [md5] - UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file - DISCREPANCY: nintendo-snes/sgb2.boot.rom: RetroDECK says OK but supersnes9x says size mismatch: got 524288, accepted [256] - supersnes9x [size] - Core files: 1495 in pack, 0 required missing, 1 optional missing -``` - -`--verbose` also lists the passing entries that an emulator validates, with -the check and the source lines behind it: - -``` -Recalbox: 351/351 OK [md5] - OK (required): macintosh/MacII.ROM - minivmac validates size=262144 [minivmac/cfg/CNFGRAPI.h:32, minivmac/src/OSGLUERETRO.c:781] -``` - -### Cross-reference section - -After per-file results, `verify.py` prints a cross-reference report. This lists -files that emulator cores need but that the platform YAML doesn't declare. These -files are still included in packs automatically, but the report helps identify -gaps in platform coverage data. - -The cross-reference uses `resolve_platform_cores()` to determine which emulator -profiles are relevant for each platform, then checks whether each profile's files -appear in the platform config. - -### Filtering output - -```bash -# By platform -python scripts/verify.py --platform batocera - -# By emulator core -python scripts/verify.py --emulator beetle_psx - -# By system -python scripts/verify.py --system sony-playstation - -# By hardware target -python scripts/verify.py --platform retroarch --target switch - -# JSON for scripted processing: the entries that need attention -python scripts/verify.py --platform retroarch --json -# every entry, passing ones included, with its ground truth -python scripts/verify.py --platform retroarch --json --verbose -``` +- [Reading this project's own reports](reading-reports.md): `verify.py` output, + status codes and severity levels +- [What the packs do not hold](../gaps.md)