fix: hold the probes to the repository checks

The new probes now refuse abbreviated options, read the GitHub token through github_headers, type every signature and document every option. The site build watches region_check.py and the content list, and validate.yml's tests/** counts as watching the list. The testing guide names the two new test modules.

.github/workflows/deploy-site.yml

@@ -40,6 +40,7 @@ on:
       - "scripts/provenance_report.py"
       - "scripts/refresh_data_dirs.py"
       - "scripts/region.py"
+      - "scripts/region_check.py"
       - "scripts/release_record.py"
       - "scripts/restore_large_files.py"
       - "scripts/safeparse.py"
@@ -72,6 +73,8 @@ on:
       - "release.json"
       - "derivable.json"
       - "measurements/**"
+      # The coverage table and the region results read the test game list.
+      - "tests/campaign/content.yml"
       # The Starlight project: config, components, styles, lockfile. Its
       # generated folders are gitignored and never trigger anything.
       - "website/**"

scripts/region_check.py

@@ -342,18 +342,19 @@ def collect(root: Path = ROOT, stored_records: list | None = None) -> list[dict]
 
 
 def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     sub = parser.add_subparsers(dest="command", required=True)
-    go = sub.add_parser("run", help="the four runs of each region game, recorded")
+    go = sub.add_parser("run", allow_abbrev=False, help="the four runs of each region game, recorded")
     go.add_argument("--emulator", required=True, help="the profile, e.g. beetle_psx")
     go.add_argument("--system", required=True, help="the system, e.g. sony-playstation")
     go.add_argument("--core-name", help="buildbot core name, e.g. mednafen_psx (downloads the nightly)")
     go.add_argument("--core", help="a core .so to use instead of the nightly")
     go.add_argument("--core-artifact", help="where --core was downloaded from")
-    go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content"))
+    go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content"),
+                    help="where romfetch keeps the test games (default: .cache/test-content)")
     go.add_argument("--variants", nargs="*", choices=VARIANTS, help="a subset of the four runs")
     go.add_argument("--timeout", type=int, default=900, help="seconds per run")
-    sub.add_parser("report", help="the outcomes the records hold, as JSON")
+    sub.add_parser("report", allow_abbrev=False, help="the outcomes the records hold, as JSON")
     args = parser.parse_args(argv)
     if args.command == "run":
         if not (args.core or args.core_name):

scripts/standalone_probe.py

@@ -44,6 +44,7 @@ import urllib.request
 from collections.abc import Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import IO
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
@@ -198,8 +199,8 @@ def emulator_settings(emu: Emulator, bios_dir: Path, overrides: Sequence[tuple[s
 
 
 def parse_xwd(data: bytes) -> tuple[int, int, bytes]:
-    """Width, height and RGB pixels of an XWD image, as Xvfb's -fbdir
-    keeps its screen: 32 bits per pixel, least significant byte first."""
+    """Width, height and RGB pixels of an XWD image, the file Xvfb keeps
+    its screen in: 32 bits per pixel, least significant byte first."""
     if len(data) < 100:
         raise cp.ProbeError("framebuffer file shorter than an XWD header")
     fields = struct.unpack_from(">25I", data, 0)
@@ -223,11 +224,10 @@ def parse_xwd(data: bytes) -> tuple[int, int, bytes]:
 # --------------------------------------------------------------------------
 
 def github_releases(repo: str) -> list[dict]:
+    from scraper.base_scraper import github_headers  # noqa: PLC0415
+
     request = urllib.request.Request(f"https://api.github.com/repos/{repo}/releases",
-                                     headers={"Accept": "application/vnd.github+json"})
-    token = os.environ.get("GITHUB_TOKEN")
-    if token:
-        request.add_header("Authorization", f"Bearer {token}")
+                                     headers=github_headers())
     with urllib.request.urlopen(request, timeout=60) as response:
         return json.load(response)
 
@@ -293,11 +293,12 @@ def now_iso() -> str:
 
 # The X server runs inside the sandbox, on the sandbox's own /tmp: the
 # display is private to the run, and the host's /tmp/.X11-unix (read-only
-# under WSLg) is never touched. -fbdir keeps the screen as an XWD file in the
-# run's scratch folder, which the probe reads from outside.
+# under WSLg) is never touched. Its fbdir option keeps the screen as an XWD
+# file in the run's scratch folder, which the probe reads from outside.
 DISPLAY_NUMBER = 1
 XVFB_WRAPPER = (
-    f'Xvfb :{DISPLAY_NUMBER} -screen 0 {SCREEN[0]}x{SCREEN[1]}x24 -nolisten tcp -fbdir "$1" -noreset >/dev/null 2>&1 &\n'
+    f'Xvfb ":{DISPLAY_NUMBER}" -screen 0 {SCREEN[0]}x{SCREEN[1]}x24 -nolisten tcp -fbdir "$1" '
+    '-noreset >/dev/null 2>&1 &\n'
     'i=0\n'
     f'while [ ! -S /tmp/.X11-unix/X{DISPLAY_NUMBER} ]; do\n'
     '  i=$((i + 1)); [ "$i" -gt 400 ] && exit 97; sleep 0.05\n'
@@ -322,7 +323,7 @@ def stop(proc: subprocess.Popen) -> None:
 ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
 
 
-def read_lines(stream, out: list[dict], samples: list) -> None:
+def read_lines(stream: IO[bytes], out: list[dict], samples: list) -> None:
     """The emulator's console, line by line, each with the sample count
     reached when it was written, colour escapes removed."""
     for raw in iter(stream.readline, b""):
@@ -632,9 +633,9 @@ def build_parser() -> argparse.ArgumentParser:
     for name, text in (("resolve", "print the asset the platform's installer picks"),
                        ("fetch", "download and unpack it into .cache/standalone/")):
         p = sub.add_parser(name, help=text)
-        p.add_argument("emulator", choices=sorted(EMULATORS))
+        p.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name")
     common = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
-    common.add_argument("emulator", choices=sorted(EMULATORS))
+    common.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name")
     common.add_argument("--content", required=True, help="the game: a .cue, .iso or .chd")
     common.add_argument("--content-label", help="identity of the content, its Redump or No-Intro name")
     common.add_argument("--bios-file", action="append", default=[], metavar="DEST=SRC",

tests/test_standalone_probe.py

@@ -17,7 +17,7 @@ import standalone_probe as sp  # noqa: E402
 
 
 def xwd(width: int, height: int, pixels: bytes, ncolors: int = 0, pad: int = 0) -> bytes:
-    """An XWD file as Xvfb -fbdir writes it: version 7, 32 bpp, LSB first."""
+    """An XWD file as Xvfb keeps its screen: version 7, 32 bpp, LSB first."""
     name = b"Xvfb main window\0"
     header_size = 100 + len(name)
     bytes_per_line = width * 4 + pad
@@ -105,7 +105,7 @@ class Settings(unittest.TestCase):
         argv = sp.with_display(["/emu", "--", "game.cue"], Path("/fb"))
         self.assertEqual(argv[:2], ["/bin/sh", "-c"])
         self.assertEqual(argv[3:], ["xvfb", "/fb", "/emu", "--", "game.cue"])
-        self.assertIn(f"Xvfb :{sp.DISPLAY_NUMBER}", argv[2])
+        self.assertIn(f'Xvfb ":{sp.DISPLAY_NUMBER}"', argv[2])
 
 
 class Oracle(unittest.TestCase):

tests/test_workflow_paths.py

@@ -154,7 +154,11 @@ class ThePullRequestBuildsTheSite(unittest.TestCase):
             if not path.startswith(("scripts/", ".github/"))
         }
         self.assertIn("wiki/**", inputs)
-        self.assertEqual(sorted(inputs - watched), [])
+        # A path the deploy names inside a folder validate.yml watches
+        # whole (tests/campaign/content.yml under tests/**) is watched.
+        unwatched = [path for path in inputs - watched
+                     if not any(fnmatch.fnmatch(path, pattern) for pattern in watched)]
+        self.assertEqual(sorted(unwatched), [])
         self.assertIn("scripts/**", watched)
 
 

wiki/testing-guide.md

@@ -59,6 +59,8 @@ Every module in `tests/`, by the area it guards.
 | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |
 | Measurement records | synthetic record files, the committed `measurements/` against HEAD | `test_measurements` |
 | Real-condition campaign | synthetic profiles, records and content folders, a fake host for load, disk, clock and commands, and whole passes against a libretro core built from C when a compiler and strace exist | `test_campaign` |
+| Region checks | the variants a region game runs under, the outcome read from synthetic records, the system page sentence and the coverage table, and that every system whose BIOS a profile ties to Japan, the US and Europe has a game per region in the content list or a reason | `test_region_check` |
+| Standalone emulators | the release asset EmuDeck's rule picks, the AppImage offset, the Xvfb framebuffer reader, the settings written and the oracle the verdicts use, with no emulator run | `test_standalone_probe` |
 | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |

wiki/tools.md

@@ -1032,6 +1032,14 @@ flock ~/.local/state/retrobios/probe.lock python scripts/region_check.py run \
 python scripts/region_check.py report        # the outcome of every region record, as JSON
 ```
 
+| Option | Effect |
+|--------|--------|
+| `--core-name` | the buildbot name of the core, whose x86_64 nightly is downloaded once into `.cache/region/cores/` and named in the record |
+| `--core`, `--core-artifact` | a core `.so` to use instead, and where it was downloaded from |
+| `--content-dir` | where romfetch keeps the test games (default `.cache/test-content`) |
+| `--variants` | a subset of `full`, `own`, `others` and `pack` |
+| `--timeout` | seconds per run (default 900) |
+
 The file the core opened in the `full` run, read from strace, settles the
 claim: `confirmed` when its `region:` holds the game's region, `contradicted`
 when it does not, `unread` when the core opened none of them. Each variant's
@@ -1073,6 +1081,17 @@ flock ~/.local/state/retrobios/probe.lock python scripts/standalone_probe.py che
 | Files | `strace -f` inside the sandbox: the files the emulator opened in the staged BIOS folder, and the ones it looked for there and did not find |
 | Verdict | `core_probe.judge` on each repetition, with the oracle of `tests/campaign/oracle.yml` looked up by the content's sha1 and the emulator's name, plus the emulator's own error strings this script lists (DuckStation's "No BIOS image found for", PCSX2's "Unable to auto locate a BIOS image"). An emulator that exits before the last sample is a crash |
 
+| Option | Effect |
+|--------|--------|
+| `--bios-file DEST=SRC` | copy SRC into the staged BIOS folder as DEST |
+| `--setting SECTION.KEY=VALUE` | an emulator setting written before the run |
+| `--sha256` | the fetched asset to run (default: the newest fetched) |
+| `--samples`, `--interval` | screen samples per repetition (default 60), and seconds between two (default 1) |
+| `--oracle` | the boot oracle file (default `tests/campaign/oracle.yml`, as `core_probe.py`) |
+| `--png-dir` | each distinct sampled screen of the first repetition as PNG |
+| `--record`, `--subject`, `--supersedes`, `--max-load`, `--work-dir`, `--no-trace`, `--require-sandbox` | as in `core_probe.py` |
+| `check`: `--vary`, `--observable`, `--claim`, `--method`, `--repeat` | as in `core_probe.py`, compared on `verdict` by default |
+
 Supported: DuckStation and PCSX2. Dolphin, Flycast, PPSSPP, melonDS, xemu and
 MAME are installed by EmuDeck from Flathub, Ryujinx from a Codeberg tarball and
 RPCS3 from its own update server: each needs its installer's rule before it