fix: let region runs wait for a busy host

core_probe refuses to record above its load threshold. region_check gave up on the first refusal, so twelve runs ended at once on a host still busy from the previous run. It now tries a refused run again every 15 seconds for up to --max-load-wait seconds, the threshold unchanged.

scripts/region_check.py

@@ -31,6 +31,7 @@ import re
 import subprocess
 import sys
 import tempfile
+import time
 import urllib.request
 from collections.abc import Callable, Iterable
 from pathlib import Path
@@ -267,6 +268,26 @@ def local_resolver() -> Callable[[dict], Path | None]:
     return local
 
 
+BUSY = 3  # core_probe's exit status when it refuses to record on a busy host
+POLL_S = 15
+
+
+def probe_when_idle(argv: list[str], max_wait: float,
+                    clock: Callable[[], float] = time.monotonic,
+                    pause: Callable[[float], None] = time.sleep,
+                    runner: Callable[..., subprocess.CompletedProcess] = subprocess.run
+                    ) -> subprocess.CompletedProcess:
+    """core_probe, run again while it refuses a busy host, as the campaign
+    does: every POLL_S seconds until max_wait seconds have passed. The load
+    threshold stays core_probe's own."""
+    started = clock()
+    while True:
+        proc = runner(argv, cwd=ROOT, capture_output=True, text=True)
+        if proc.returncode != BUSY or clock() - started >= max_wait:
+            return proc
+        pause(POLL_S)
+
+
 def run(args: argparse.Namespace) -> int:
     from common import load_emulator_profiles  # noqa: PLC0415
 
@@ -299,7 +320,7 @@ def run(args: argparse.Namespace) -> int:
                     argv += ["--core-artifact", url]
                 for dest, src in sorted(variants[label].items()):
                     argv += ["--system-file", f"{dest}={src}"]
-                proc = subprocess.run(argv, cwd=ROOT, capture_output=True, text=True)
+                proc = probe_when_idle(argv, args.max_load_wait)
                 status = "recorded" if proc.returncode == 0 else f"exit {proc.returncode}"
                 print(json.dumps({"game": game["name"], "variant": label, "status": status,
                                   "error": proc.stderr.strip()[-400:] if proc.returncode else None}),
@@ -354,6 +375,8 @@ def main(argv: list[str] | None = None) -> int:
                     help="where romfetch keeps the test games (default: .cache/test-content)")
     go.add_argument("--variants", nargs="*", choices=VARIANTS, help="a subset of the four runs")
     go.add_argument("--timeout", type=int, default=900, help="seconds per run")
+    go.add_argument("--max-load-wait", type=int, default=1800,
+                    help="seconds a run waits for a busy host to settle before it is given up (default 1800)")
     sub.add_parser("report", allow_abbrev=False, help="the outcomes the records hold, as JSON")
     args = parser.parse_args(argv)
     if args.command == "run":

tests/test_region_check.py

@@ -59,6 +59,35 @@ class Entries(unittest.TestCase):
         self.assertRegex(slug, rc.SUBJECT)
 
 
+class BusyHost(unittest.TestCase):
+    def _run(self, codes: list[int], max_wait: float) -> tuple[int, int]:
+        import subprocess
+
+        now = [0.0]
+        calls = []
+
+        def runner(argv, **_kwargs):
+            calls.append(argv)
+            return subprocess.CompletedProcess(argv, codes[min(len(calls), len(codes)) - 1], "", "")
+
+        def pause(seconds: float) -> None:
+            now[0] += seconds
+
+        proc = rc.probe_when_idle(["probe"], max_wait, clock=lambda: now[0], pause=pause, runner=runner)
+        return proc.returncode, len(calls)
+
+    def test_a_refused_run_is_tried_again_once_the_host_settles(self) -> None:
+        self.assertEqual(self._run([rc.BUSY, rc.BUSY, 0], 600), (0, 3))
+
+    def test_it_gives_up_after_the_wait(self) -> None:
+        code, calls = self._run([rc.BUSY], 60)
+        self.assertEqual(code, rc.BUSY)
+        self.assertEqual(calls, 60 // rc.POLL_S + 1)
+
+    def test_another_failure_is_not_retried(self) -> None:
+        self.assertEqual(self._run([1, 0], 600), (1, 1))
+
+
 class Outcome(unittest.TestCase):
     ENTRIES = rc.region_entries(PROFILE, "sony-playstation")
     ALL = ["scph5500.bin", "scph5501.bin", "scph5502.bin"]

wiki/tools.md

@@ -1096,6 +1096,7 @@ python scripts/region_check.py report        # the outcome of every region recor
 | `--content-dir` | where romfetch keeps the test games (default `.cache/test-content`) |
 | `--variants` | a subset of `full`, `own`, `others` and `pack` |
 | `--timeout` | seconds per run (default 900) |
+| `--max-load-wait` | seconds a run refused on a busy host is tried again, every 15 seconds, before it is given up (default 1800). The load threshold stays core_probe's |
 
 The file the core opened in the `full` run, read from strace, settles the
 claim: `confirmed` when its `region:` holds the game's region, `contradicted`