diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py new file mode 100644 index 00000000..19cf7281 --- /dev/null +++ b/tests/test_doc_surfaces.py @@ -0,0 +1,119 @@ +"""What the wiki and the generated README state holds against the code. + +Each class reads a page and the code it describes. A page that quotes a +function's return shape, a resolution order or a field list drifts when the +code moves, and nothing else notices. +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import io +import json +import re +import sys +import unittest +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WIKI = ROOT / "wiki" +sys.path.insert(0, str(ROOT / "scripts")) + +from tests.test_prose_style import _python_prose, _sentence_case # noqa: E402 + +FENCED = re.compile(r"^```.*?^```", re.S | re.M) +PUBLISHED = sorted(WIKI.glob("*.md")) + [ROOT / "README.md", ROOT / "CONTRIBUTING.md"] + + +def _page(name: str) -> str: + return (WIKI / name).read_text(encoding="utf-8") + + +def _section(text: str, heading: str) -> str: + """Body of a markdown section, up to the next heading of any level.""" + start = re.search(rf"(?m)^#+ {re.escape(heading)}\n", text) + if start is None: + raise AssertionError(f"no section {heading!r}") + rest = text[start.end():] + blanked = FENCED.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), rest) + end = re.search(r"(?m)^#+ ", blanked) + return rest[: end.start()] if end else rest + + +def _table_rows(text: str) -> list[list[str]]: + rows = [] + for line in text.splitlines(): + if line.startswith("|") and not re.match(r"\|\s*-", line): + rows.append([cell.strip() for cell in line.strip().strip("|").split(" | ")]) + return rows[1:] + + +class TestSkeletonReadsTheResultShape(unittest.TestCase): + """The skeleton indexed verify_platform's result as a list, and a test + copied from it stopped on KeyError: 0 before any assertion.""" + + def test_every_key_it_reads_is_one_verify_platform_returns(self): + from verify import verify_platform + + result = verify_platform( + {"platform": "x", "verification_mode": "existence", "systems": {}}, + {"files": {}, "indexes": {}}, + emu_profiles={}, + ) + text = _page("testing-guide.md").split("Example skeleton:", 1)[1] + code = re.search(r"```python\n(.*?)```", text, re.S).group(1) + read = [ + node.slice + for node in ast.walk(ast.parse(code)) + if isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == "result" + ] + self.assertTrue(read, "the skeleton no longer reads the result") + for key in read: + self.assertIsInstance(key, ast.Constant) + self.assertIn(key.value, result) + + +class NoHedgedRules(unittest.TestCase): + """The profiling guide and the verification discipline turned rules into + wishes ("it's worth", "Ideally", "something to look into").""" + + FILLER = re.compile( + r"(?i)\bit'?s worth\b|\bworth noting\b|\bit helps to\b|^Ideally\b" + r"|\bsomething to look into\b|\busually indicate\b" + ) + + def test_published_pages(self): + offenders = [ + f"{page.name}: {line[:80]}" + for page in PUBLISHED + for line in FENCED.sub("", page.read_text(encoding="utf-8")).splitlines() + if self.FILLER.search(line) + ] + self.assertEqual(offenders, [], "\n".join(offenders)) + + +class ProfileFieldsRequired(unittest.TestCase): + """The table marked source, profiled_date, core_version and systems as + required and called the schema authoritative, which requires none.""" + + def test_yes_matches_the_schema(self): + schema = json.loads((ROOT / "schemas" / "emulator.schema.json").read_text(encoding="utf-8")) + required = set(schema["required"]) + for rule in schema.get("allOf", []): + for branch in ("then", "else"): + required |= set(rule.get(branch, {}).get("required", [])) + section = _section(_page("profiling.md"), "Profile fields") + marked = { + row[0].strip("`") for row in _table_rows(section) + if row[1].startswith(("yes", "for `type")) + } + self.assertEqual(marked, required) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index d0d17163..db8249f7 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -55,17 +55,15 @@ a system_dir lookup) are expected. What matters are file additions the port introduces, files the port dropped, or hash values that differ between the two codebases. -If the source is hosted outside GitHub, it's worth exploring further. Emulator -source on GitLab, Codeberg, SourceForge, Bitbucket, archive.org -snapshots, and community mirror tarballs. Inspecting copyright headers -or license strings in the libretro fork often points to the original -author's site. The upstream code exists somewhere, so it's worth continuing the search before concluding the source is unavailable. +Source hosted outside GitHub is searched on GitLab, Codeberg, SourceForge, +Bitbucket, archive.org snapshots and community mirror tarballs. Copyright +headers and license strings in the libretro fork often name the original +author's site. A source not found yet is a search not finished, and the +profile waits for it. -One thing worth noting: even when the same repository was analyzed for -a related profile (e.g., fbneo for arcade systems), it helps to do a -fresh pass for each new profile. When fbneo_neogeo was profiled, the -NeoGeo subset referenced BIOS files that the main arcade analysis -hadn't encountered. A fresh look avoids carrying over blind spots. +Each profile gets its own reading of the code, even when the same repository +was read for a related profile (`fbneo`, then `fbneo_neogeo`). The earlier +reading says what that profile loads, not what the new one loads. A declared upstream that answers 404 is not proof the project is gone. The account can differ from the Android package id (`aenu.aps3e` lives at @@ -467,19 +465,23 @@ even if documentation mentions it. ### Profile fields -The authoritative version of this table is `schemas/emulator.schema.json`, -which CI validates every profile against. +`schemas/emulator.schema.json` is the contract CI validates every profile +against. It requires `emulator` and `type`, then `files`, or `alias_of` for an +alias. A profile also carries `source_commit` or `profiled_date`: without +either, profile_sync has no revision to read the `source_ref` lines at and +reports the profile as uncheckable, and the CI job that runs +`profile_sync.py --strict` on each profile a pull request adds fails on it. | Field | Required | Description | |-------|----------|-------------| | `emulator` | yes | display name | | `type` | yes | `libretro`, `standalone`, `standalone + libretro`, `alias`, `launcher`, `game`, `utility`, `test` | | `core_classification` | no | `pure_libretro`, `official_port`, `community_fork`, `frozen_snapshot`, `enhanced_fork`, `game_engine`, `embedded_hle`, `launcher`, `alias`, `other` | -| `source` | yes | libretro core repository URL. A dict when the core ships under several repos | +| `source` | no | libretro core repository URL. A dict when the core ships under several repos | | `upstream` | no | original emulator repository URL | -| `profiled_date` | yes | date of source analysis. Quote it, or YAML parses it as a date object | -| `core_version` | yes | version analyzed | -| `source_commit` | no | commit SHA of `source` the code was read at. Anchors every `source_ref` line number | +| `profiled_date` | see above | date of source analysis. Quote it, or YAML parses it as a date object | +| `core_version` | no | version analyzed | +| `source_commit` | see above | commit SHA of `source` the code was read at. Anchors every `source_ref` line number | | `upstream_commit` | no | same, for `upstream` | | `source_branch` | no | branch `source` was read from when it is not the default one (a port living on a pull request branch). profile_sync follows it | | `built_by` | no | platforms that ship this build under its core name, compiled by them or installed from the project's own release (Recalbox's `mame0278`, the GSplus legacy build). On those platforms the name resolves to this profile, never through `all_libretro`: the buildbot does not publish it | @@ -487,7 +489,7 @@ which CI validates every profile against. | `bios_directory` | no | the folder an agnostic core scans, relative to the system directory (`pcsx2/bios/`). A scan never seeds from files outside it | | `display_name` | no | full display name (e.g. "Sega - Mega Drive (BlastEm)") | | `logo` | no | image URL used on the emulator page | -| `systems` | yes | list of system IDs this core handles | +| `systems` | no | list of system IDs this core handles | | `cores` | no | every upstream name the core is known by, for buildbot/target matching | | `alias_of` | for `type: alias` | profile key this one aliases. The alias carries no `files` | | `mode` | no | default mode: `standalone`, `libretro`, or `both` | diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index bf4c3c1d..070a8a09 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -1,7 +1,5 @@ # Testing guide -This page covers how to run, understand, and extend the test suite. - The suite is discovered dynamically across focused modules, so its exact count is reported by the test runner instead of being hand-maintained here. Tests do not require network access. Most build synthetic fixtures under the repository's @@ -46,7 +44,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_doc_surfaces`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture @@ -193,17 +191,19 @@ script could have established, the script gains that check in the same change. Example skeleton: ```python -def test_42_my_new_behavior(self): - # Write minimal fixtures to self.root - profile = {"emulator": "test_core", "files": [...]} +def test_my_new_behavior(self): + # Minimal fixtures under self.root + profile = {"emulator": "test_core", "type": "libretro", "files": [...]} with open(os.path.join(self.emulators_dir, "test_core.yml"), "w") as f: yaml.dump(profile, f) - # Call production code - result = verify_platform(self.config, self.db, ...) + # The entry point the CLI uses + config = load_platform_config("test_md5", self.platforms_dir) + result = verify_platform(config, self.db, self.emulators_dir) - # Assert specific outcomes - self.assertEqual(result[0]["status"], Status.OK) + # One outcome, read from the per-file details + by_name = {d["name"]: d for d in result["details"]} + self.assertEqual(by_name["my_file.bin"]["status"], Status.OK) ``` ### test_pack_integrity.py @@ -242,9 +242,13 @@ The test suite is one layer of verification. The full quality gate is: 5. Pack integrity passes (every declared file extractable with correct hash) 6. `mkdocs build --strict` and `python scripts/validate_site.py` pass -If a change passes tests but breaks the pipeline, it's worth investigating before merging. Similarly, new CRITICAL entries in the verify output after a change usually indicate something to look into. The pipeline is designed so that all steps agree: if verify reports N files for a platform, the pack should contain exactly N files. +A change that passes the tests but breaks the pipeline is not merged. A new +CRITICAL entry in the verify output after a change is explained before the +change lands. The steps agree with each other: when verify reports N files for +a platform, the pack holds exactly N files. -Ideally, tests, code, and documentation ship together. When profiles and platform configs are involved, updating them in the same change helps keep everything in sync. +Tests, code, profiles, platform configs and documentation ship in the same +change. A run that dies with a segmentation fault or an exception the code cannot raise is run again once, alone, before anything is investigated. Only a