mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
docs: align the testing and profiling guides
The test skeleton indexed verify_platform's dict result as a list and stopped on KeyError: 0. The verification discipline and the profiling guide hedged rules the project holds as firm. The profile field table marked as required fields the schema does not require, and now names the pin that profile_sync --strict needs.
This commit is contained in:
Abdessamad Derraz committed
1 parent
348c56e94c
commit
32bda05be3
3 files changed
+154
-29
No files matched your search
@@ -0,0 +1,119 @@
|
||||
"""What the wiki and the generated README state holds against the code.
|
||||
|
||||
Each class reads a page and the code it describes. A page that quotes a
|
||||
function's return shape, a resolution order or a field list drifts when the
|
||||
code moves, and nothing else notices.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import inspect
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
WIKI = ROOT / "wiki"
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from tests.test_prose_style import _python_prose, _sentence_case # noqa: E402
|
||||
|
||||
FENCED = re.compile(r"^```.*?^```", re.S | re.M)
|
||||
PUBLISHED = sorted(WIKI.glob("*.md")) + [ROOT / "README.md", ROOT / "CONTRIBUTING.md"]
|
||||
|
||||
|
||||
def _page(name: str) -> str:
|
||||
return (WIKI / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _section(text: str, heading: str) -> str:
|
||||
"""Body of a markdown section, up to the next heading of any level."""
|
||||
start = re.search(rf"(?m)^#+ {re.escape(heading)}\n", text)
|
||||
if start is None:
|
||||
raise AssertionError(f"no section {heading!r}")
|
||||
rest = text[start.end():]
|
||||
blanked = FENCED.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), rest)
|
||||
end = re.search(r"(?m)^#+ ", blanked)
|
||||
return rest[: end.start()] if end else rest
|
||||
|
||||
|
||||
def _table_rows(text: str) -> list[list[str]]:
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith("|") and not re.match(r"\|\s*-", line):
|
||||
rows.append([cell.strip() for cell in line.strip().strip("|").split(" | ")])
|
||||
return rows[1:]
|
||||
|
||||
|
||||
class TestSkeletonReadsTheResultShape(unittest.TestCase):
|
||||
"""The skeleton indexed verify_platform's result as a list, and a test
|
||||
copied from it stopped on KeyError: 0 before any assertion."""
|
||||
|
||||
def test_every_key_it_reads_is_one_verify_platform_returns(self):
|
||||
from verify import verify_platform
|
||||
|
||||
result = verify_platform(
|
||||
{"platform": "x", "verification_mode": "existence", "systems": {}},
|
||||
{"files": {}, "indexes": {}},
|
||||
emu_profiles={},
|
||||
)
|
||||
text = _page("testing-guide.md").split("Example skeleton:", 1)[1]
|
||||
code = re.search(r"```python\n(.*?)```", text, re.S).group(1)
|
||||
read = [
|
||||
node.slice
|
||||
for node in ast.walk(ast.parse(code))
|
||||
if isinstance(node, ast.Subscript)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "result"
|
||||
]
|
||||
self.assertTrue(read, "the skeleton no longer reads the result")
|
||||
for key in read:
|
||||
self.assertIsInstance(key, ast.Constant)
|
||||
self.assertIn(key.value, result)
|
||||
|
||||
|
||||
class NoHedgedRules(unittest.TestCase):
|
||||
"""The profiling guide and the verification discipline turned rules into
|
||||
wishes ("it's worth", "Ideally", "something to look into")."""
|
||||
|
||||
FILLER = re.compile(
|
||||
r"(?i)\bit'?s worth\b|\bworth noting\b|\bit helps to\b|^Ideally\b"
|
||||
r"|\bsomething to look into\b|\busually indicate\b"
|
||||
)
|
||||
|
||||
def test_published_pages(self):
|
||||
offenders = [
|
||||
f"{page.name}: {line[:80]}"
|
||||
for page in PUBLISHED
|
||||
for line in FENCED.sub("", page.read_text(encoding="utf-8")).splitlines()
|
||||
if self.FILLER.search(line)
|
||||
]
|
||||
self.assertEqual(offenders, [], "\n".join(offenders))
|
||||
|
||||
|
||||
class ProfileFieldsRequired(unittest.TestCase):
|
||||
"""The table marked source, profiled_date, core_version and systems as
|
||||
required and called the schema authoritative, which requires none."""
|
||||
|
||||
def test_yes_matches_the_schema(self):
|
||||
schema = json.loads((ROOT / "schemas" / "emulator.schema.json").read_text(encoding="utf-8"))
|
||||
required = set(schema["required"])
|
||||
for rule in schema.get("allOf", []):
|
||||
for branch in ("then", "else"):
|
||||
required |= set(rule.get(branch, {}).get("required", []))
|
||||
section = _section(_page("profiling.md"), "Profile fields")
|
||||
marked = {
|
||||
row[0].strip("`") for row in _table_rows(section)
|
||||
if row[1].startswith(("yes", "for `type"))
|
||||
}
|
||||
self.assertEqual(marked, required)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+19
-17
@@ -55,17 +55,15 @@ a system_dir lookup) are expected. What matters are file additions the
|
||||
port introduces, files the port dropped, or hash values that differ
|
||||
between the two codebases.
|
||||
|
||||
If the source is hosted outside GitHub, it's worth exploring further. Emulator
|
||||
source on GitLab, Codeberg, SourceForge, Bitbucket, archive.org
|
||||
snapshots, and community mirror tarballs. Inspecting copyright headers
|
||||
or license strings in the libretro fork often points to the original
|
||||
author's site. The upstream code exists somewhere, so it's worth continuing the search before concluding the source is unavailable.
|
||||
Source hosted outside GitHub is searched on GitLab, Codeberg, SourceForge,
|
||||
Bitbucket, archive.org snapshots and community mirror tarballs. Copyright
|
||||
headers and license strings in the libretro fork often name the original
|
||||
author's site. A source not found yet is a search not finished, and the
|
||||
profile waits for it.
|
||||
|
||||
One thing worth noting: even when the same repository was analyzed for
|
||||
a related profile (e.g., fbneo for arcade systems), it helps to do a
|
||||
fresh pass for each new profile. When fbneo_neogeo was profiled, the
|
||||
NeoGeo subset referenced BIOS files that the main arcade analysis
|
||||
hadn't encountered. A fresh look avoids carrying over blind spots.
|
||||
Each profile gets its own reading of the code, even when the same repository
|
||||
was read for a related profile (`fbneo`, then `fbneo_neogeo`). The earlier
|
||||
reading says what that profile loads, not what the new one loads.
|
||||
|
||||
A declared upstream that answers 404 is not proof the project is gone. The
|
||||
account can differ from the Android package id (`aenu.aps3e` lives at
|
||||
@@ -467,19 +465,23 @@ even if documentation mentions it.
|
||||
|
||||
### Profile fields
|
||||
|
||||
The authoritative version of this table is `schemas/emulator.schema.json`,
|
||||
which CI validates every profile against.
|
||||
`schemas/emulator.schema.json` is the contract CI validates every profile
|
||||
against. It requires `emulator` and `type`, then `files`, or `alias_of` for an
|
||||
alias. A profile also carries `source_commit` or `profiled_date`: without
|
||||
either, profile_sync has no revision to read the `source_ref` lines at and
|
||||
reports the profile as uncheckable, and the CI job that runs
|
||||
`profile_sync.py --strict` on each profile a pull request adds fails on it.
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `emulator` | yes | display name |
|
||||
| `type` | yes | `libretro`, `standalone`, `standalone + libretro`, `alias`, `launcher`, `game`, `utility`, `test` |
|
||||
| `core_classification` | no | `pure_libretro`, `official_port`, `community_fork`, `frozen_snapshot`, `enhanced_fork`, `game_engine`, `embedded_hle`, `launcher`, `alias`, `other` |
|
||||
| `source` | yes | libretro core repository URL. A dict when the core ships under several repos |
|
||||
| `source` | no | libretro core repository URL. A dict when the core ships under several repos |
|
||||
| `upstream` | no | original emulator repository URL |
|
||||
| `profiled_date` | yes | date of source analysis. Quote it, or YAML parses it as a date object |
|
||||
| `core_version` | yes | version analyzed |
|
||||
| `source_commit` | no | commit SHA of `source` the code was read at. Anchors every `source_ref` line number |
|
||||
| `profiled_date` | see above | date of source analysis. Quote it, or YAML parses it as a date object |
|
||||
| `core_version` | no | version analyzed |
|
||||
| `source_commit` | see above | commit SHA of `source` the code was read at. Anchors every `source_ref` line number |
|
||||
| `upstream_commit` | no | same, for `upstream` |
|
||||
| `source_branch` | no | branch `source` was read from when it is not the default one (a port living on a pull request branch). profile_sync follows it |
|
||||
| `built_by` | no | platforms that ship this build under its core name, compiled by them or installed from the project's own release (Recalbox's `mame0278`, the GSplus legacy build). On those platforms the name resolves to this profile, never through `all_libretro`: the buildbot does not publish it |
|
||||
@@ -487,7 +489,7 @@ which CI validates every profile against.
|
||||
| `bios_directory` | no | the folder an agnostic core scans, relative to the system directory (`pcsx2/bios/`). A scan never seeds from files outside it |
|
||||
| `display_name` | no | full display name (e.g. "Sega - Mega Drive (BlastEm)") |
|
||||
| `logo` | no | image URL used on the emulator page |
|
||||
| `systems` | yes | list of system IDs this core handles |
|
||||
| `systems` | no | list of system IDs this core handles |
|
||||
| `cores` | no | every upstream name the core is known by, for buildbot/target matching |
|
||||
| `alias_of` | for `type: alias` | profile key this one aliases. The alias carries no `files` |
|
||||
| `mode` | no | default mode: `standalone`, `libretro`, or `both` |
|
||||
|
||||
+16
-12
@@ -1,7 +1,5 @@
|
||||
# Testing guide
|
||||
|
||||
This page covers how to run, understand, and extend the test suite.
|
||||
|
||||
The suite is discovered dynamically across focused modules, so its exact count
|
||||
is reported by the test runner instead of being hand-maintained here. Tests do
|
||||
not require network access. Most build synthetic fixtures under the repository's
|
||||
@@ -46,7 +44,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
|
||||
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
|
||||
| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_doc_surfaces`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
|
||||
## Test architecture
|
||||
|
||||
@@ -193,17 +191,19 @@ script could have established, the script gains that check in the same change.
|
||||
Example skeleton:
|
||||
|
||||
```python
|
||||
def test_42_my_new_behavior(self):
|
||||
# Write minimal fixtures to self.root
|
||||
profile = {"emulator": "test_core", "files": [...]}
|
||||
def test_my_new_behavior(self):
|
||||
# Minimal fixtures under self.root
|
||||
profile = {"emulator": "test_core", "type": "libretro", "files": [...]}
|
||||
with open(os.path.join(self.emulators_dir, "test_core.yml"), "w") as f:
|
||||
yaml.dump(profile, f)
|
||||
|
||||
# Call production code
|
||||
result = verify_platform(self.config, self.db, ...)
|
||||
# The entry point the CLI uses
|
||||
config = load_platform_config("test_md5", self.platforms_dir)
|
||||
result = verify_platform(config, self.db, self.emulators_dir)
|
||||
|
||||
# Assert specific outcomes
|
||||
self.assertEqual(result[0]["status"], Status.OK)
|
||||
# One outcome, read from the per-file details
|
||||
by_name = {d["name"]: d for d in result["details"]}
|
||||
self.assertEqual(by_name["my_file.bin"]["status"], Status.OK)
|
||||
```
|
||||
|
||||
### test_pack_integrity.py
|
||||
@@ -242,9 +242,13 @@ The test suite is one layer of verification. The full quality gate is:
|
||||
5. Pack integrity passes (every declared file extractable with correct hash)
|
||||
6. `mkdocs build --strict` and `python scripts/validate_site.py` pass
|
||||
|
||||
If a change passes tests but breaks the pipeline, it's worth investigating before merging. Similarly, new CRITICAL entries in the verify output after a change usually indicate something to look into. The pipeline is designed so that all steps agree: if verify reports N files for a platform, the pack should contain exactly N files.
|
||||
A change that passes the tests but breaks the pipeline is not merged. A new
|
||||
CRITICAL entry in the verify output after a change is explained before the
|
||||
change lands. The steps agree with each other: when verify reports N files for
|
||||
a platform, the pack holds exactly N files.
|
||||
|
||||
Ideally, tests, code, and documentation ship together. When profiles and platform configs are involved, updating them in the same change helps keep everything in sync.
|
||||
Tests, code, profiles, platform configs and documentation ship in the same
|
||||
change.
|
||||
|
||||
A run that dies with a segmentation fault or an exception the code cannot
|
||||
raise is run again once, alone, before anything is investigated. Only a
|
||||
|
||||
Reference in new issue
Block a user