From f9f688ae6e5b1784a188c3b142be5dd97d552039 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:05:10 +0100 Subject: [PATCH] fix: hold the probes to the repository checks The new probes now refuse abbreviated options, read the GitHub token through github_headers, type every signature and document every option. The site build watches region_check.py and the content list, and validate.yml's tests/** counts as watching the list. The testing guide names the two new test modules. --- .github/workflows/deploy-site.yml | 3 +++ scripts/region_check.py | 9 +++++---- scripts/standalone_probe.py | 25 +++++++++++++------------ tests/test_standalone_probe.py | 4 ++-- tests/test_workflow_paths.py | 6 +++++- wiki/testing-guide.md | 2 ++ wiki/tools.md | 19 +++++++++++++++++++ 7 files changed, 49 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index f73a52a08..553a95ff0 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -40,6 +40,7 @@ on: - "scripts/provenance_report.py" - "scripts/refresh_data_dirs.py" - "scripts/region.py" + - "scripts/region_check.py" - "scripts/release_record.py" - "scripts/restore_large_files.py" - "scripts/safeparse.py" @@ -72,6 +73,8 @@ on: - "release.json" - "derivable.json" - "measurements/**" + # The coverage table and the region results read the test game list. + - "tests/campaign/content.yml" # The Starlight project: config, components, styles, lockfile. Its # generated folders are gitignored and never trigger anything. - "website/**" diff --git a/scripts/region_check.py b/scripts/region_check.py index 668f4caea..795a3cef1 100644 --- a/scripts/region_check.py +++ b/scripts/region_check.py @@ -342,18 +342,19 @@ def collect(root: Path = ROOT, stored_records: list | None = None) -> list[dict] def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) - go = sub.add_parser("run", help="the four runs of each region game, recorded") + go = sub.add_parser("run", allow_abbrev=False, help="the four runs of each region game, recorded") go.add_argument("--emulator", required=True, help="the profile, e.g. beetle_psx") go.add_argument("--system", required=True, help="the system, e.g. sony-playstation") go.add_argument("--core-name", help="buildbot core name, e.g. mednafen_psx (downloads the nightly)") go.add_argument("--core", help="a core .so to use instead of the nightly") go.add_argument("--core-artifact", help="where --core was downloaded from") - go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content")) + go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content"), + help="where romfetch keeps the test games (default: .cache/test-content)") go.add_argument("--variants", nargs="*", choices=VARIANTS, help="a subset of the four runs") go.add_argument("--timeout", type=int, default=900, help="seconds per run") - sub.add_parser("report", help="the outcomes the records hold, as JSON") + sub.add_parser("report", allow_abbrev=False, help="the outcomes the records hold, as JSON") args = parser.parse_args(argv) if args.command == "run": if not (args.core or args.core_name): diff --git a/scripts/standalone_probe.py b/scripts/standalone_probe.py index 6a8b397a2..65294ef4a 100644 --- a/scripts/standalone_probe.py +++ b/scripts/standalone_probe.py @@ -44,6 +44,7 @@ import urllib.request from collections.abc import Sequence from dataclasses import dataclass, field from pathlib import Path +from typing import IO sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -198,8 +199,8 @@ def emulator_settings(emu: Emulator, bios_dir: Path, overrides: Sequence[tuple[s def parse_xwd(data: bytes) -> tuple[int, int, bytes]: - """Width, height and RGB pixels of an XWD image, as Xvfb's -fbdir - keeps its screen: 32 bits per pixel, least significant byte first.""" + """Width, height and RGB pixels of an XWD image, the file Xvfb keeps + its screen in: 32 bits per pixel, least significant byte first.""" if len(data) < 100: raise cp.ProbeError("framebuffer file shorter than an XWD header") fields = struct.unpack_from(">25I", data, 0) @@ -223,11 +224,10 @@ def parse_xwd(data: bytes) -> tuple[int, int, bytes]: # -------------------------------------------------------------------------- def github_releases(repo: str) -> list[dict]: + from scraper.base_scraper import github_headers # noqa: PLC0415 + request = urllib.request.Request(f"https://api.github.com/repos/{repo}/releases", - headers={"Accept": "application/vnd.github+json"}) - token = os.environ.get("GITHUB_TOKEN") - if token: - request.add_header("Authorization", f"Bearer {token}") + headers=github_headers()) with urllib.request.urlopen(request, timeout=60) as response: return json.load(response) @@ -293,11 +293,12 @@ def now_iso() -> str: # The X server runs inside the sandbox, on the sandbox's own /tmp: the # display is private to the run, and the host's /tmp/.X11-unix (read-only -# under WSLg) is never touched. -fbdir keeps the screen as an XWD file in the -# run's scratch folder, which the probe reads from outside. +# under WSLg) is never touched. Its fbdir option keeps the screen as an XWD +# file in the run's scratch folder, which the probe reads from outside. DISPLAY_NUMBER = 1 XVFB_WRAPPER = ( - f'Xvfb :{DISPLAY_NUMBER} -screen 0 {SCREEN[0]}x{SCREEN[1]}x24 -nolisten tcp -fbdir "$1" -noreset >/dev/null 2>&1 &\n' + f'Xvfb ":{DISPLAY_NUMBER}" -screen 0 {SCREEN[0]}x{SCREEN[1]}x24 -nolisten tcp -fbdir "$1" ' + '-noreset >/dev/null 2>&1 &\n' 'i=0\n' f'while [ ! -S /tmp/.X11-unix/X{DISPLAY_NUMBER} ]; do\n' ' i=$((i + 1)); [ "$i" -gt 400 ] && exit 97; sleep 0.05\n' @@ -322,7 +323,7 @@ def stop(proc: subprocess.Popen) -> None: ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") -def read_lines(stream, out: list[dict], samples: list) -> None: +def read_lines(stream: IO[bytes], out: list[dict], samples: list) -> None: """The emulator's console, line by line, each with the sample count reached when it was written, colour escapes removed.""" for raw in iter(stream.readline, b""): @@ -632,9 +633,9 @@ def build_parser() -> argparse.ArgumentParser: for name, text in (("resolve", "print the asset the platform's installer picks"), ("fetch", "download and unpack it into .cache/standalone/")): p = sub.add_parser(name, help=text) - p.add_argument("emulator", choices=sorted(EMULATORS)) + p.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name") common = argparse.ArgumentParser(add_help=False, allow_abbrev=False) - common.add_argument("emulator", choices=sorted(EMULATORS)) + common.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name") common.add_argument("--content", required=True, help="the game: a .cue, .iso or .chd") common.add_argument("--content-label", help="identity of the content, its Redump or No-Intro name") common.add_argument("--bios-file", action="append", default=[], metavar="DEST=SRC", diff --git a/tests/test_standalone_probe.py b/tests/test_standalone_probe.py index ba63d54a3..73763f648 100644 --- a/tests/test_standalone_probe.py +++ b/tests/test_standalone_probe.py @@ -17,7 +17,7 @@ import standalone_probe as sp # noqa: E402 def xwd(width: int, height: int, pixels: bytes, ncolors: int = 0, pad: int = 0) -> bytes: - """An XWD file as Xvfb -fbdir writes it: version 7, 32 bpp, LSB first.""" + """An XWD file as Xvfb keeps its screen: version 7, 32 bpp, LSB first.""" name = b"Xvfb main window\0" header_size = 100 + len(name) bytes_per_line = width * 4 + pad @@ -105,7 +105,7 @@ class Settings(unittest.TestCase): argv = sp.with_display(["/emu", "--", "game.cue"], Path("/fb")) self.assertEqual(argv[:2], ["/bin/sh", "-c"]) self.assertEqual(argv[3:], ["xvfb", "/fb", "/emu", "--", "game.cue"]) - self.assertIn(f"Xvfb :{sp.DISPLAY_NUMBER}", argv[2]) + self.assertIn(f'Xvfb ":{sp.DISPLAY_NUMBER}"', argv[2]) class Oracle(unittest.TestCase): diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py index f9929b4bf..f2a11ee2c 100644 --- a/tests/test_workflow_paths.py +++ b/tests/test_workflow_paths.py @@ -154,7 +154,11 @@ class ThePullRequestBuildsTheSite(unittest.TestCase): if not path.startswith(("scripts/", ".github/")) } self.assertIn("wiki/**", inputs) - self.assertEqual(sorted(inputs - watched), []) + # A path the deploy names inside a folder validate.yml watches + # whole (tests/campaign/content.yml under tests/**) is watched. + unwatched = [path for path in inputs - watched + if not any(fnmatch.fnmatch(path, pattern) for pattern in watched)] + self.assertEqual(sorted(unwatched), []) self.assertIn("scripts/**", watched) diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 84c1bd8cf..5635b2888 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -59,6 +59,8 @@ Every module in `tests/`, by the area it guards. | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` | | Measurement records | synthetic record files, the committed `measurements/` against HEAD | `test_measurements` | | Real-condition campaign | synthetic profiles, records and content folders, a fake host for load, disk, clock and commands, and whole passes against a libretro core built from C when a compiler and strace exist | `test_campaign` | +| Region checks | the variants a region game runs under, the outcome read from synthetic records, the system page sentence and the coverage table, and that every system whose BIOS a profile ties to Japan, the US and Europe has a game per region in the content list or a reason | `test_region_check` | +| Standalone emulators | the release asset EmuDeck's rule picks, the AppImage offset, the Xvfb framebuffer reader, the settings written and the oracle the verdicts use, with no emulator run | `test_standalone_probe` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | diff --git a/wiki/tools.md b/wiki/tools.md index 3a92361f7..fb963ed75 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1032,6 +1032,14 @@ flock ~/.local/state/retrobios/probe.lock python scripts/region_check.py run \ python scripts/region_check.py report # the outcome of every region record, as JSON ``` +| Option | Effect | +|--------|--------| +| `--core-name` | the buildbot name of the core, whose x86_64 nightly is downloaded once into `.cache/region/cores/` and named in the record | +| `--core`, `--core-artifact` | a core `.so` to use instead, and where it was downloaded from | +| `--content-dir` | where romfetch keeps the test games (default `.cache/test-content`) | +| `--variants` | a subset of `full`, `own`, `others` and `pack` | +| `--timeout` | seconds per run (default 900) | + The file the core opened in the `full` run, read from strace, settles the claim: `confirmed` when its `region:` holds the game's region, `contradicted` when it does not, `unread` when the core opened none of them. Each variant's @@ -1073,6 +1081,17 @@ flock ~/.local/state/retrobios/probe.lock python scripts/standalone_probe.py che | Files | `strace -f` inside the sandbox: the files the emulator opened in the staged BIOS folder, and the ones it looked for there and did not find | | Verdict | `core_probe.judge` on each repetition, with the oracle of `tests/campaign/oracle.yml` looked up by the content's sha1 and the emulator's name, plus the emulator's own error strings this script lists (DuckStation's "No BIOS image found for", PCSX2's "Unable to auto locate a BIOS image"). An emulator that exits before the last sample is a crash | +| Option | Effect | +|--------|--------| +| `--bios-file DEST=SRC` | copy SRC into the staged BIOS folder as DEST | +| `--setting SECTION.KEY=VALUE` | an emulator setting written before the run | +| `--sha256` | the fetched asset to run (default: the newest fetched) | +| `--samples`, `--interval` | screen samples per repetition (default 60), and seconds between two (default 1) | +| `--oracle` | the boot oracle file (default `tests/campaign/oracle.yml`, as `core_probe.py`) | +| `--png-dir` | each distinct sampled screen of the first repetition as PNG | +| `--record`, `--subject`, `--supersedes`, `--max-load`, `--work-dir`, `--no-trace`, `--require-sandbox` | as in `core_probe.py` | +| `check`: `--vary`, `--observable`, `--claim`, `--method`, `--repeat` | as in `core_probe.py`, compared on `verdict` by default | + Supported: DuckStation and PCSX2. Dolphin, Flycast, PPSSPP, melonDS, xemu and MAME are installed by EmuDeck from Flathub, Ryujinx from a Codeberg tarball and RPCS3 from its own update server: each needs its installer's rule before it