feat: rescrape retrobat at its pinned commit

The scraper read emulatorLauncher master on every run, so an extractor
change was diffed together with upstream's commits since the pin. It now
rereads the commit platforms/retrobat.yml pins. --revision master moves
it, and check_freshness and refresh_stale pass that through the
registry's rescrape_args.

The extractors also read master (f681f435): Kronos and YabaSanshiro
fall back to sega_101.bin then mpr-17933.bin when the chosen BIOS is
absent, Ymir writes a forced BIOS only under its override flag, and a
folder created below bios (Ymir's cdb) is a passes_path record. At the
pin the 187 records are unchanged.

scripts/scraper/base_scraper.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import inspect
 import json
 import re
 import sys
@@ -391,10 +392,16 @@ def scraper_cli(
     parser.add_argument("--dry-run", action="store_true", help="Show scraped data")
     parser.add_argument("--output", "-o", help="Output YAML file")
     parser.add_argument("--json", action="store_true", help="Output as JSON")
+    takes_revision = "revision" in inspect.signature(scraper_class).parameters
+    if takes_revision:
+        parser.add_argument(
+            "--revision",
+            help="Read this branch head or commit instead of the revision the platform file pins",
+        )
     args = parser.parse_args()
 
     try:
-        scraper = scraper_class()
+        scraper = scraper_class(revision=args.revision) if takes_revision else scraper_class()
         reqs = scraper.fetch_requirements()
     except (OSError, ValueError, RuntimeError) as e:
         _exit_on_scrape_error(e)

scripts/scraper/retrobat_launchers.py

@@ -1868,13 +1868,27 @@ def _root_writes(src: Source, emulator: str) -> list[Record]:
         if root_reach is None:
             continue
         method = root_reach[0]
+        # A folder below the root, `string x = Path.Combine(root, "cdb")`
+        # that the method creates or tests as a directory. A file below it
+        # is a guarded write's, read by _guarded_write.
+        folders = {root.group(1): ("bios", root)}
+        for sub in re.compile(
+            rf"string (\w+) = Path\.Combine\({root.group(1)}, ((?:{STRING}, )*{STRING})\);"
+        ).finditer(src.text, root.end(), method[1]):
+            if re.search(
+                rf"\b(?:\w*CreateDirectory|Directory\.Exists)\({sub.group(1)}\)",
+                src.text[sub.end() : method[1]],
+            ):
+                folders[sub.group(1)] = ("/".join(["bios", *_strings(sub.group(2))]), sub)
+        names = "|".join(map(re.escape, folders))
         for write in re.compile(
-            rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", (?:"\'" \+ )?{root.group(1)}(?: \+ "\'")?\);'
+            rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", (?:"\'" \+ )?({names})(?: \+ "\'")?\);'
         ).finditer(src.text, root.end(), method[1]):
             reached = _conditions(src, write.start())
             if reached is None:
                 continue
-            writer, section, key = write.groups()
+            writer, section, key, held = write.groups()
+            folder, declared = folders[held]
             target = f"[{section}] {key}" if section else key
             records.append(
                 _record(
@@ -1888,12 +1902,14 @@ def _root_writes(src: Source, emulator: str) -> list[Record]:
                             [
                                 *reached[2],
                                 src.lines(root.start(), root.end()),
+                                *([src.lines(declared.start(), declared.end())]
+                                  if declared is not root else []),
                                 src.lines(write.start(), write.end()),
                             ],
                         )
                     ),
-                    search_dir="bios",
-                    passes_to=f"{_config_name(src, method, writer)} {target} = bios",
+                    search_dir=folder,
+                    passes_to=f"{_config_name(src, method, writer)} {target} = {folder}",
                     fails_when_missing=False,
                     depends_on_option=", ".join(reached[1]) or None,
                 )
@@ -2376,11 +2392,7 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record
     gate = src.find(
         rf'if \(SystemConfig\.isOptSet\("{option}"\)', f"{option} gate", method[0], override.start()
     )
-    gate_block = src.block_at(gate.end())
-    if not gate_block[0] < override.start() < gate_block[1]:
-        raise LauncherPatternMissing(
-            src.path, f"{option} block around line {src.line(override.start())}"
-        )
+    gate_block = _gate_span(src, gate, override)
     write = re.compile(
         rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", ([^;\n]*\b{var}\b[^;\n]*)\);'
     ).search(code, override.end(), method[1])
@@ -2390,6 +2402,8 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record
     guarded = re.compile(rf"if \(File\.Exists\({var}\)[^\n]*\)\s*$").search(
         code, method[0], write.start()
     )
+    flag = None if guarded else _flag_guard(code, var, method[0], write.start())
+    guarded = guarded or flag
     skipped = re.findall(
         rf'SystemConfig\["{option}"\] != "(\w+)"', code[gate.start() : write.end()]
     )
@@ -2397,8 +2411,13 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record
     target = f"{config} [{section}] {key}" if section else f"{config} {key}"
     suffix = ", only if the file exists" if guarded else ", written without an existence check"
     fails = f"{key} not written" if guarded else "path written without an existence check"
-    write_span = src.lines(_statement_start(src, write.start()), write.end())
+    write_span = src.lines(
+        flag.start() if flag else _statement_start(src, write.start()), write.end()
+    )
     choices, choice_lines = feature_choices(features, emulator, option)
+    fallback = _fallback_names(src, var, gate_block[1], write.start())
+    if fallback is not None:
+        fails = f"{', then '.join(fallback[0])}, instead, else {fails}"
     records = []
     default = list(
         re.compile(rf'string {var} = Path\.Combine\({_BIOS_ROOT}, "([^"]+)"\);').finditer(
@@ -2437,9 +2456,71 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record
             depends_on_option=option,
         )
     )
+    if fallback is not None:
+        names, span = fallback
+        unset = f"{default[-1].group(1)} absent with {option} unset" if default else None
+        chosen = f"the {option} file absent" + (
+            f", {option} not {' or '.join(skipped)}" if skipped else ""
+        )
+        records.append(
+            _record(
+                emulator,
+                "standalone",
+                "names_searched",
+                names,
+                _path_parts(src, span, write_span),
+                case_insensitive=True,
+                search_dir="bios",
+                passes_to=f"{target} = <first found>{suffix}",
+                fails_when_missing=f"{key} not written" if guarded else fails,
+                depends_on_option=" or ".join(c for c in (unset, chosen) if c),
+            )
+        )
     return records + _root_writes(src, emulator)
 
 
+def _gate_span(src: Source, gate: re.Match[str], override: re.Match[str]) -> tuple[int, int]:
+    """The span an option gate governs: its block, or the one statement a
+    braceless if runs, which must be the override."""
+    paren = gate.start() + len("if ")
+    after = paren + len(_call_args(src, paren)) + 2
+    if re.match(r"\s*\{", src.text[after:]):
+        block = src.block_at(after)
+    else:
+        block = (after, _statement_end(src, after))
+    if not block[0] < override.start() < block[1]:
+        raise LauncherPatternMissing(
+            src.path, f"option block around line {src.line(override.start())}"
+        )
+    return block
+
+
+def _flag_guard(code: str, var: str, start: int, end: int) -> re.Match[str] | None:
+    """A write under `if (flag)` where flag holds File.Exists of the path."""
+    flag = re.compile(rf"bool (\w+) = [^;\n]*\bFile\.Exists\({var}\)[^;\n]*;").search(
+        code, start, end
+    )
+    if flag is None or not re.compile(rf"if \({flag.group(1)}\)\s*$", re.M).search(
+        code, flag.end(), end
+    ):
+        return None
+    return flag
+
+
+def _fallback_names(
+    src: Source, var: str, start: int, end: int
+) -> tuple[list[str], tuple[int, int]] | None:
+    """Names tried in bios, first existing one kept, when the chosen file is absent."""
+    found = re.compile(
+        rf"if \(([^\n]*!File\.Exists\({var}\)[^\n]*)\)\s*"
+        rf"{var} = new\[\] \{{ (?P<names>(?:{STRING}(?:, )?)+) \}}\.Select\((?P<arg>\w+) => "
+        rf"Path\.Combine\({_BIOS_ROOT}, (?P=arg)\)\)\.FirstOrDefault\(File\.Exists\) \?\? {var};"
+    ).search(_code(src), start, end)
+    if found is None:
+        return None
+    return _strings(found.group("names")), src.lines(found.start(), found.end())
+
+
 OPTION_PATHS = {
     f"{GENERATORS}/Kronos.Generator.cs": "kronos",
     f"{GENERATORS}/SSF.Generator.cs": "ssf",

scripts/scraper/retrobat_scraper.py

@@ -12,6 +12,7 @@ import json
 import re
 import urllib.error
 import urllib.request
+from pathlib import Path
 
 try:
     from .base_scraper import (
@@ -39,6 +40,7 @@ except ImportError:
     from retrobat_launchers import extract_launchers
 
 PLATFORM_NAME = "retrobat"
+PLATFORM_FILE = Path(__file__).resolve().parents[2] / "platforms" / "retrobat.yml"
 
 LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher"
 SOURCE_URL = (
@@ -132,6 +134,28 @@ def launcher_revision(url: str) -> str:
     return found.group(1)
 
 
+def pinned_revision(platform_file: Path = PLATFORM_FILE) -> str:
+    """The emulatorLauncher commit platforms/retrobat.yml was read from.
+
+    A rescrape reads this commit unless told to move, so a change to an
+    extractor shows in the diff alone, without upstream's commits since.
+    """
+    from common import yaml_load
+
+    with platform_file.open(encoding="utf-8") as fh:
+        source = str((yaml_load(fh) or {}).get("source") or "")
+    return launcher_revision(source)
+
+
+def resolve_revision(revision: str | None) -> str:
+    """The commit to read: the pin when None, a branch's head, or a sha."""
+    if revision is None:
+        return pinned_revision()
+    if re.fullmatch(r"[0-9a-f]{40}", revision):
+        return revision
+    return fetch_branch_head(LAUNCHER_REPO, revision)
+
+
 def fetch_launchers(rev: str) -> list[dict]:
     """Launcher records read from the generators at the commit the BIOS list comes from."""
     texts = {
@@ -160,10 +184,14 @@ SYSTEM_SLUG_MAP = {
 class Scraper(BaseScraper):
     """Scraper for RetroBat batocera-systems.json."""
 
-    def __init__(self, url: str | None = None) -> None:
-        # batocera-systems.json is read from master: the commit read is recorded.
+    def __init__(self, url: str | None = None, revision: str | None = None) -> None:
+        # The BIOS list and the generators are read at one commit, the one
+        # platforms/retrobat.yml pins unless a revision moves it (--revision
+        # master reads the branch head). The commit read is recorded.
         if url is None:
-            url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master"))
+            url = SOURCE_URL.format(rev=resolve_revision(revision))
+        elif revision is not None:
+            raise ValueError("pass a url or a revision, not both")
         super().__init__(url=url)
         self._parsed: dict | None = None