From 01ca6253785b317a61431a182a813458e02e25b8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:40:05 +0100 Subject: [PATCH 1/6] feat: rescrape retrobat at its pinned commit The scraper read emulatorLauncher master on every run, so an extractor change was diffed together with upstream's commits since the pin. It now rereads the commit platforms/retrobat.yml pins. --revision master moves it, and check_freshness and refresh_stale pass that through the registry's rescrape_args. The extractors also read master (f681f435): Kronos and YabaSanshiro fall back to sega_101.bin then mpr-17933.bin when the chosen BIOS is absent, Ymir writes a forced BIOS only under its override flag, and a folder created below bios (Ymir's cdb) is a passes_path record. At the pin the 187 records are unchanged. --- platforms/_registry.yml | 3 + scripts/check_freshness.py | 20 ++++- scripts/refresh_stale.py | 1 + scripts/scraper/base_scraper.py | 9 ++- scripts/scraper/retrobat_launchers.py | 101 +++++++++++++++++++++--- scripts/scraper/retrobat_scraper.py | 34 +++++++- tests/test_launchers_retrobat.py | 107 ++++++++++++++++++++++++++ tests/test_refresh_stale.py | 11 +++ tests/test_scraper_contract.py | 23 +++++- 9 files changed, 290 insertions(+), 19 deletions(-) diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 6c5babac7..84c599a51 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -585,6 +585,9 @@ platforms: extract_to: bios/ logo: https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png scraper: retrobat + # The scraper rereads the emulatorLauncher commit retrobat.yml pins; + # check_freshness and refresh_stale move it to the branch head. + rescrape_args: ["--revision", "master"] source_url: https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/master/batocera-systems/Resources/batocera-systems.json source_format: json hash_type: md5 diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index 4ef9e145c..ae0e27596 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -282,7 +282,8 @@ def diff_targets(old: dict, new: dict) -> PlatformDiff: def _scrape_into_copy( - module: str, source: Path, workdir: Path, timeout: int = 900 + module: str, source: Path, workdir: Path, timeout: int = 900, + extra: tuple[str, ...] = (), ) -> tuple[Path, str | None]: """Run a scraper's own write path on a throwaway copy of *source*. @@ -292,7 +293,7 @@ def _scrape_into_copy( target = workdir / source.name shutil.copyfile(source, target) proc = subprocess.run( - [sys.executable, "-m", module, "-o", str(target)], + [sys.executable, "-m", module, "-o", str(target), *extra], cwd=REPO_ROOT, capture_output=True, text=True, @@ -327,6 +328,17 @@ def _scrapable_platforms(platforms_dir: Path) -> list[tuple[str, str, Path]]: return rows +def rescrape_args(platforms_dir: Path, name: str) -> tuple[str, ...]: + """What a rescrape passes so the scraper reads upstream's moving head. + + A scraper that rereads the revision its platform file pins by default + (RetroBat) names the ref to move to in the registry's rescrape_args. + """ + with (platforms_dir / "_registry.yml").open(encoding="utf-8") as fh: + registry = (yaml_load(fh) or {}).get("platforms") or {} + return tuple(str(a) for a in (registry.get(name) or {}).get("rescrape_args") or ()) + + def check_platforms(platforms_dir: Path, workdir: Path, jobs: int) -> list[Finding]: rows = _scrapable_platforms(platforms_dir) findings: list[Finding] = [] @@ -334,7 +346,9 @@ def check_platforms(platforms_dir: Path, workdir: Path, jobs: int) -> list[Findi def run(row: tuple[str, str, Path]) -> Finding: name, module, path = row try: - fresh, error = _scrape_into_copy(module, path, workdir) + fresh, error = _scrape_into_copy( + module, path, workdir, extra=rescrape_args(platforms_dir, name) + ) except subprocess.TimeoutExpired: return Finding("platforms", name, ERROR, detail="scraper timed out") if error: diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index b50e472d8..c4c4fe37e 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -380,6 +380,7 @@ def _command_for( f"scripts.scraper.{scraper}_scraper", "-o", f"platforms/{subject}.yml", + *(str(a) for a in entry.get("rescrape_args") or ()), ] if area == "targets": diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py index bbd8dc220..dcda281de 100644 --- a/scripts/scraper/base_scraper.py +++ b/scripts/scraper/base_scraper.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import json import re import sys @@ -391,10 +392,16 @@ def scraper_cli( parser.add_argument("--dry-run", action="store_true", help="Show scraped data") parser.add_argument("--output", "-o", help="Output YAML file") parser.add_argument("--json", action="store_true", help="Output as JSON") + takes_revision = "revision" in inspect.signature(scraper_class).parameters + if takes_revision: + parser.add_argument( + "--revision", + help="Read this branch head or commit instead of the revision the platform file pins", + ) args = parser.parse_args() try: - scraper = scraper_class() + scraper = scraper_class(revision=args.revision) if takes_revision else scraper_class() reqs = scraper.fetch_requirements() except (OSError, ValueError, RuntimeError) as e: _exit_on_scrape_error(e) diff --git a/scripts/scraper/retrobat_launchers.py b/scripts/scraper/retrobat_launchers.py index a49709194..a443f83c3 100644 --- a/scripts/scraper/retrobat_launchers.py +++ b/scripts/scraper/retrobat_launchers.py @@ -1868,13 +1868,27 @@ def _root_writes(src: Source, emulator: str) -> list[Record]: if root_reach is None: continue method = root_reach[0] + # A folder below the root, `string x = Path.Combine(root, "cdb")` + # that the method creates or tests as a directory. A file below it + # is a guarded write's, read by _guarded_write. + folders = {root.group(1): ("bios", root)} + for sub in re.compile( + rf"string (\w+) = Path\.Combine\({root.group(1)}, ((?:{STRING}, )*{STRING})\);" + ).finditer(src.text, root.end(), method[1]): + if re.search( + rf"\b(?:\w*CreateDirectory|Directory\.Exists)\({sub.group(1)}\)", + src.text[sub.end() : method[1]], + ): + folders[sub.group(1)] = ("/".join(["bios", *_strings(sub.group(2))]), sub) + names = "|".join(map(re.escape, folders)) for write in re.compile( - rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", (?:"\'" \+ )?{root.group(1)}(?: \+ "\'")?\);' + rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", (?:"\'" \+ )?({names})(?: \+ "\'")?\);' ).finditer(src.text, root.end(), method[1]): reached = _conditions(src, write.start()) if reached is None: continue - writer, section, key = write.groups() + writer, section, key, held = write.groups() + folder, declared = folders[held] target = f"[{section}] {key}" if section else key records.append( _record( @@ -1888,12 +1902,14 @@ def _root_writes(src: Source, emulator: str) -> list[Record]: [ *reached[2], src.lines(root.start(), root.end()), + *([src.lines(declared.start(), declared.end())] + if declared is not root else []), src.lines(write.start(), write.end()), ], ) ), - search_dir="bios", - passes_to=f"{_config_name(src, method, writer)} {target} = bios", + search_dir=folder, + passes_to=f"{_config_name(src, method, writer)} {target} = {folder}", fails_when_missing=False, depends_on_option=", ".join(reached[1]) or None, ) @@ -2376,11 +2392,7 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record gate = src.find( rf'if \(SystemConfig\.isOptSet\("{option}"\)', f"{option} gate", method[0], override.start() ) - gate_block = src.block_at(gate.end()) - if not gate_block[0] < override.start() < gate_block[1]: - raise LauncherPatternMissing( - src.path, f"{option} block around line {src.line(override.start())}" - ) + gate_block = _gate_span(src, gate, override) write = re.compile( rf'(\w+)\.WriteValue\("([^"]*)", "([^"]+)", ([^;\n]*\b{var}\b[^;\n]*)\);' ).search(code, override.end(), method[1]) @@ -2390,6 +2402,8 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record guarded = re.compile(rf"if \(File\.Exists\({var}\)[^\n]*\)\s*$").search( code, method[0], write.start() ) + flag = None if guarded else _flag_guard(code, var, method[0], write.start()) + guarded = guarded or flag skipped = re.findall( rf'SystemConfig\["{option}"\] != "(\w+)"', code[gate.start() : write.end()] ) @@ -2397,8 +2411,13 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record target = f"{config} [{section}] {key}" if section else f"{config} {key}" suffix = ", only if the file exists" if guarded else ", written without an existence check" fails = f"{key} not written" if guarded else "path written without an existence check" - write_span = src.lines(_statement_start(src, write.start()), write.end()) + write_span = src.lines( + flag.start() if flag else _statement_start(src, write.start()), write.end() + ) choices, choice_lines = feature_choices(features, emulator, option) + fallback = _fallback_names(src, var, gate_block[1], write.start()) + if fallback is not None: + fails = f"{', then '.join(fallback[0])}, instead, else {fails}" records = [] default = list( re.compile(rf'string {var} = Path\.Combine\({_BIOS_ROOT}, "([^"]+)"\);').finditer( @@ -2437,9 +2456,71 @@ def _option_path_in(src: Source, features: Source, emulator: str) -> list[Record depends_on_option=option, ) ) + if fallback is not None: + names, span = fallback + unset = f"{default[-1].group(1)} absent with {option} unset" if default else None + chosen = f"the {option} file absent" + ( + f", {option} not {' or '.join(skipped)}" if skipped else "" + ) + records.append( + _record( + emulator, + "standalone", + "names_searched", + names, + _path_parts(src, span, write_span), + case_insensitive=True, + search_dir="bios", + passes_to=f"{target} = {suffix}", + fails_when_missing=f"{key} not written" if guarded else fails, + depends_on_option=" or ".join(c for c in (unset, chosen) if c), + ) + ) return records + _root_writes(src, emulator) +def _gate_span(src: Source, gate: re.Match[str], override: re.Match[str]) -> tuple[int, int]: + """The span an option gate governs: its block, or the one statement a + braceless if runs, which must be the override.""" + paren = gate.start() + len("if ") + after = paren + len(_call_args(src, paren)) + 2 + if re.match(r"\s*\{", src.text[after:]): + block = src.block_at(after) + else: + block = (after, _statement_end(src, after)) + if not block[0] < override.start() < block[1]: + raise LauncherPatternMissing( + src.path, f"option block around line {src.line(override.start())}" + ) + return block + + +def _flag_guard(code: str, var: str, start: int, end: int) -> re.Match[str] | None: + """A write under `if (flag)` where flag holds File.Exists of the path.""" + flag = re.compile(rf"bool (\w+) = [^;\n]*\bFile\.Exists\({var}\)[^;\n]*;").search( + code, start, end + ) + if flag is None or not re.compile(rf"if \({flag.group(1)}\)\s*$", re.M).search( + code, flag.end(), end + ): + return None + return flag + + +def _fallback_names( + src: Source, var: str, start: int, end: int +) -> tuple[list[str], tuple[int, int]] | None: + """Names tried in bios, first existing one kept, when the chosen file is absent.""" + found = re.compile( + rf"if \(([^\n]*!File\.Exists\({var}\)[^\n]*)\)\s*" + rf"{var} = new\[\] \{{ (?P(?:{STRING}(?:, )?)+) \}}\.Select\((?P\w+) => " + rf"Path\.Combine\({_BIOS_ROOT}, (?P=arg)\)\)\.FirstOrDefault\(File\.Exists\) \?\? {var};" + ).search(_code(src), start, end) + if found is None: + return None + return _strings(found.group("names")), src.lines(found.start(), found.end()) + + OPTION_PATHS = { f"{GENERATORS}/Kronos.Generator.cs": "kronos", f"{GENERATORS}/SSF.Generator.cs": "ssf", diff --git a/scripts/scraper/retrobat_scraper.py b/scripts/scraper/retrobat_scraper.py index 7ae5a605a..f5e7cc78d 100644 --- a/scripts/scraper/retrobat_scraper.py +++ b/scripts/scraper/retrobat_scraper.py @@ -12,6 +12,7 @@ import json import re import urllib.error import urllib.request +from pathlib import Path try: from .base_scraper import ( @@ -39,6 +40,7 @@ except ImportError: from retrobat_launchers import extract_launchers PLATFORM_NAME = "retrobat" +PLATFORM_FILE = Path(__file__).resolve().parents[2] / "platforms" / "retrobat.yml" LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher" SOURCE_URL = ( @@ -132,6 +134,28 @@ def launcher_revision(url: str) -> str: return found.group(1) +def pinned_revision(platform_file: Path = PLATFORM_FILE) -> str: + """The emulatorLauncher commit platforms/retrobat.yml was read from. + + A rescrape reads this commit unless told to move, so a change to an + extractor shows in the diff alone, without upstream's commits since. + """ + from common import yaml_load + + with platform_file.open(encoding="utf-8") as fh: + source = str((yaml_load(fh) or {}).get("source") or "") + return launcher_revision(source) + + +def resolve_revision(revision: str | None) -> str: + """The commit to read: the pin when None, a branch's head, or a sha.""" + if revision is None: + return pinned_revision() + if re.fullmatch(r"[0-9a-f]{40}", revision): + return revision + return fetch_branch_head(LAUNCHER_REPO, revision) + + def fetch_launchers(rev: str) -> list[dict]: """Launcher records read from the generators at the commit the BIOS list comes from.""" texts = { @@ -160,10 +184,14 @@ SYSTEM_SLUG_MAP = { class Scraper(BaseScraper): """Scraper for RetroBat batocera-systems.json.""" - def __init__(self, url: str | None = None) -> None: - # batocera-systems.json is read from master: the commit read is recorded. + def __init__(self, url: str | None = None, revision: str | None = None) -> None: + # The BIOS list and the generators are read at one commit, the one + # platforms/retrobat.yml pins unless a revision moves it (--revision + # master reads the branch head). The commit read is recorded. if url is None: - url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master")) + url = SOURCE_URL.format(rev=resolve_revision(revision)) + elif revision is not None: + raise ValueError("pass a url or a revision, not both") super().__init__(url=url) self._parsed: dict | None = None diff --git a/tests/test_launchers_retrobat.py b/tests/test_launchers_retrobat.py index b2d8ded7a..58dd8430e 100644 --- a/tests/test_launchers_retrobat.py +++ b/tests/test_launchers_retrobat.py @@ -11,6 +11,7 @@ from __future__ import annotations import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache import sys +import tempfile import textwrap import unittest from pathlib import Path @@ -637,6 +638,57 @@ YMIR = """ } """ +# emulatorLauncher f681f435: the default falls back to the regional BIOS. +KRONOS_FALLBACK = """ + private void SetupConfig(string path) + { + string iniFile = Path.Combine(path, "kronos.ini"); + using (var ini = new IniFile(iniFile, IniOptions.KeepEmptyValues)) + { + bool hleBios = false; + string bios = Path.Combine(AppConfig.GetFullPath("bios"), "saturn_bios.bin"); + if (SystemConfig.isOptSet("kronos_force_bios") && !string.IsNullOrEmpty(SystemConfig["kronos_force_bios"])) + { + if (SystemConfig["kronos_force_bios"] != "hle") + bios = Path.Combine(AppConfig.GetFullPath("bios"), SystemConfig["kronos_force_bios"]); + else + hleBios = true; + } + if (!hleBios && !File.Exists(bios)) + bios = new[] { "sega_101.bin", "mpr-17933.bin" }.Select(f => Path.Combine(AppConfig.GetFullPath("bios"), f)).FirstOrDefault(File.Exists) ?? bios; + + if (File.Exists(bios)) + ini.WriteValue("1.0", "General\\\\Bios", bios.Replace("\\\\", "/")); + } + } +""" + +# emulatorLauncher f681f435: a forced BIOS only, written under a flag, and +# a folder below bios for the CD block ROM. +YMIR_OVERRIDE = """ + private void SetupConfiguration(string path) + { + string iniFile = Path.Combine(path, "Ymir.toml"); + using (var ini = new IniFile(iniFile)) + { + string saturnBiosPath = null; + if (SystemConfig.isOptSet("ymir_force_bios") && !string.IsNullOrEmpty(SystemConfig["ymir_force_bios"])) + saturnBiosPath = Path.Combine(AppConfig.GetFullPath("bios"), SystemConfig["ymir_force_bios"]); + + bool overrideIpl = saturnBiosPath != null && File.Exists(saturnBiosPath); + ini.WriteValue("System.IPL", "Override", overrideIpl ? "true" : "false"); + if (overrideIpl) + ini.WriteValue("System.IPL", "Path", "'" + saturnBiosPath + "'"); + + string iplPath = AppConfig.GetFullPath("bios"); + ini.WriteValue("General.PathOverrides", "IPLROMImages", "'" + iplPath + "'"); + string cdbPath = Path.Combine(iplPath, "cdb"); + FileTools.TryCreateDirectory(cdbPath); + ini.WriteValue("General.PathOverrides", "CDBlockROMImages", "'" + cdbPath + "'"); + } + } +""" + SSF = """ private void SetupConfiguration(string path) { @@ -1043,6 +1095,46 @@ class OptionReplacesThePath(unittest.TestCase): folder["passes_to"], "Ymir.toml [General.PathOverrides] IPLROMImages = bios" ) + def test_kronos_falls_back_to_the_regional_bios(self): + path = f"{G}/Kronos.Generator.cs" + sources = _feature_sources(path, KRONOS_FALLBACK, SATURN_FEATURES) + fixed, option, fallback = rl._option_path_in( + sources[path], sources[rl.FEATURES], "kronos" + ) + self.assertEqual( + fixed["fails_when_missing"], + "sega_101.bin, then mpr-17933.bin, instead, else General\\Bios not written", + ) + self.assertEqual(option["fails_when_missing"], fixed["fails_when_missing"]) + self.assertEqual(fallback["kind"], "names_searched") + self.assertEqual(fallback["names"], ["sega_101.bin", "mpr-17933.bin"]) + self.assertEqual( + fallback["passes_to"], + "kronos.ini [1.0] General\\Bios = , only if the file exists", + ) + self.assertEqual( + fallback["depends_on_option"], + "saturn_bios.bin absent with kronos_force_bios unset" + " or the kronos_force_bios file absent, kronos_force_bios not hle", + ) + self.assertEqual(fallback["source_ref"], f"{path}:15-19") + + def test_ymir_writes_a_forced_bios_under_its_flag(self): + path = f"{G}/Ymir.Generator.cs" + sources = _feature_sources(path, YMIR_OVERRIDE, SATURN_FEATURES) + option, ipl, cdb = rl._option_path_in(sources[path], sources[rl.FEATURES], "ymir") + self.assertEqual( + option["passes_to"], + "Ymir.toml [System.IPL] Path = /, only if the file exists", + ) + self.assertEqual(option["source_ref"], f"{path}:7-13, {rl.FEATURES}:10-13") + self.assertEqual(ipl["passes_to"], "Ymir.toml [General.PathOverrides] IPLROMImages = bios") + self.assertEqual(cdb["search_dir"], "bios/cdb") + self.assertEqual( + cdb["passes_to"], "Ymir.toml [General.PathOverrides] CDBlockROMImages = bios/cdb" + ) + self.assertEqual(cdb["source_ref"], f"{path}:15-19") + def test_ssf_has_no_default_name(self): path = f"{G}/SSF.Generator.cs" sources = _feature_sources(path, SSF, SATURN_FEATURES) @@ -1488,6 +1580,21 @@ class Helpers(unittest.TestCase): src = rl.Source("x.cs", 'void f()\n{\n s = "}";\n}\n') self.assertEqual(src.lines(*src.block_at(0)), (2, 4)) + def test_a_rescrape_reads_the_pinned_commit(self): + from scraper import retrobat_scraper as rs # noqa: PLC0415 + + pin = "ca486005f598551600d04a33b3156a5d764368d6" + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as scratch: + yml = Path(scratch) / "retrobat.yml" + yml.write_text( + "source: https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/" + f"{pin}/batocera-systems/Resources/batocera-systems.json\n", + encoding="utf-8", + ) + self.assertEqual(rs.pinned_revision(yml), pin) + self.assertEqual(rs.resolve_revision(pin), pin) + self.assertEqual(rs.resolve_revision(None), rs.pinned_revision()) + def test_revision_comes_from_the_pinned_url(self): url = ( "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/" diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index b262fcc9e..8ab9ca66e 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -17,6 +17,8 @@ import unittest from pathlib import Path from unittest import mock +import yaml + REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) @@ -55,6 +57,15 @@ class Planning(unittest.TestCase): ["scripts/verify.py", "--platform", "retrobat"]], ) + def test_a_scraper_reading_its_pin_is_moved_by_rescrape_args(self): + registry = dict(REGISTRY, retrobat={"scraper": "retrobat", + "rescrape_args": ["--revision", "master"]}) + jobs, _, _ = rs.plan_jobs([finding("platforms", "retrobat")], registry, {}) + self.assertEqual(jobs[0].command[-2:], ["--revision", "master"]) + with (REPO_ROOT / "platforms" / "_registry.yml").open(encoding="utf-8") as fh: + real = yaml.safe_load(fh)["platforms"]["retrobat"] + self.assertEqual(real["rescrape_args"], ["--revision", "master"]) + def test_a_child_waits_for_its_rescraped_parent(self): """retropie inherits retroarch: run beside the parent's rescrape, its truth read the old parent and its export was refused as stale.""" diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index aab7692ca..7b588ede3 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -650,6 +650,15 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): # Read at a release tag: the pin is the tag, not a branch commit. TAG_PINNED = {"amberelec", "batocera", "bizhawk", "emuelec", "knulli", "recalbox", "rocknix", "romm"} + # Reread the commit the platform file pins, moved by the revision the + # registry's rescrape_args names: checked with that revision. + FILE_PINNED = {"retrobat"} + + @classmethod + def _construct(cls, module: object, scraper: str) -> object: + if scraper in cls.FILE_PINNED: + return module.Scraper(revision="master") + return module.Scraper() @staticmethod def _registry_scrapers() -> list[str]: @@ -671,7 +680,17 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): with self.subTest(scraper=name), mock.patch( "urllib.request.urlopen", self._refuse ), self.assertRaises(RuntimeError): - module.Scraper() + self._construct(module, scraper) + + def test_a_file_pinned_scraper_reads_its_pin_offline(self): + for scraper in sorted(self.FILE_PINNED): + module = importlib.import_module(f"scraper.{scraper}_scraper") + with self.subTest(scraper=scraper), mock.patch( + "urllib.request.urlopen", self._refuse + ): + self.assertEqual( + module.launcher_revision(module.Scraper().url), module.pinned_revision() + ) def test_a_branch_read_records_its_commit(self): """System.dat, batocera-systems.json, checkBIOS.sh, the BiosDB and the @@ -695,7 +714,7 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): name = f"scraper.{scraper}_scraper" module = importlib.import_module(name) with self.subTest(scraper=name), mock.patch("urllib.request.urlopen", answer): - instance = module.Scraper() + instance = self._construct(module, scraper) # RetroDECK reads a tree of manifests and keeps the commit apart. recorded = getattr(instance, "revision", "") or instance.url self.assertRegex(recorded, f"(^|/){sha}(/|$)") From b29500846bcf51dc09bac1106e2c9ad1fce9115a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:57:12 +0100 Subject: [PATCH 2/6] feat: read retrobat's saturn and atari builds RetroBat installs its stable archives, so each config_reads entry is read at the build the binary names: Hatari 2.4.1, hatariB 0.3, Hatari2014 at ab55c3e from the buildbot, Kronos 2.7.0 and Ymir 0.3.3. SSF ships PreviewVer R28 and YabaSanshiro 1.20.32 has no public source: binary_sha256 names the file their address refs are read in. The seven RetroBat records citing an option list now hold. --- emulators/hatari.yml | 8 ++++++++ emulators/hatari2014.yml | 11 +++++++++++ emulators/hatarib.yml | 11 +++++++++++ emulators/kronos.yml | 13 +++++++++++++ emulators/ssf.yml | 11 +++++++++++ emulators/yabasanshiro.yml | 13 +++++++++++++ emulators/ymir.yml | 16 ++++++++++++++++ schemas/emulator.schema.json | 6 ++++-- scripts/profile_sync.py | 5 ++++- scripts/validate_schemas.py | 2 +- tests/test_launcher_evidence.py | 33 +++++++++++++++++++++++++++++++++ 11 files changed, 125 insertions(+), 4 deletions(-) diff --git a/emulators/hatari.yml b/emulators/hatari.yml index 940f07f95..207a7a1f1 100644 --- a/emulators/hatari.yml +++ b/emulators/hatari.yml @@ -76,6 +76,14 @@ config_reads: keys: - key: "[ROM] szTosImageFileName" source_ref: "upstream src/configuration.c:480, upstream src/tos.c:975-986" + - file: "*.cfg" + platforms: [retrobat] + revision: "4371dcd647fc85d31c0629400adaeaa4212040d9" + revision_from: "Hatari v2.4.1, compiled Aug 3 2022, in hatari.exe of RetroBat's stable hatari.7z (2026-04-04), the v2.4.1 tag" + source_ref: "upstream src/options.c:228-229, 1099-1108, upstream src/configuration.c:967-976, 1024" + keys: + - key: "[ROM] szTosImageFileName" + source_ref: "upstream src/configuration.c:413, 884, upstream src/tos.c:942-955" notes: | The main branch of libretro/hatari is the upstream Hatari tree at diff --git a/emulators/hatari2014.yml b/emulators/hatari2014.yml index f9e2e2ecf..ff7f84be6 100644 --- a/emulators/hatari2014.yml +++ b/emulators/hatari2014.yml @@ -22,6 +22,17 @@ systems: - atari-tt - atari-falcon +config_reads: + - file: "core options" + platforms: [retrobat] + revision: "ab55c3ed0e620c91e7f059a6d3fbef7acf9bfca8" + revision_from: "library_version 1.8 ab55c3e of hatari2014_libretro.dll from the libretro buildbot windows x86_64 nightly (2026-11-10), which RetroBat installs when its own repository has no copy" + source_ref: "source libretro/libretro.c:264-300, 372, 393-413, 437-440, 984-1001, 1418, source libretro/libretro_core_options.h:129-145" + keys: + - key: "hatari_tosimage" + source_ref: "source libretro/libretro.c:617, 639-649, 1612-1617, source src/main.c:913-917, source src/tos.c:575-590" + condition: "a value other than default is a file of system/hatari/tos ending in img (libretro/libretro.c:288-299)" + notes: | Hatari 2014 is the libretro port of Hatari 1.8.0 (src/includes/version.h:16), the July 2014 release, kept on the hitari2014-mercurial branch of diff --git a/emulators/hatarib.yml b/emulators/hatarib.yml index 502f6fec2..97b3fa8b9 100644 --- a/emulators/hatarib.yml +++ b/emulators/hatarib.yml @@ -20,6 +20,17 @@ systems: - atari-tt - atari-falcon +config_reads: + - file: "core options" + platforms: [retrobat] + revision: "9c5ee608504d7e174dce99ac9237151863d78347" + revision_from: "library_version v0.3 9c5ee60 of hatarib_libretro.dll in RetroBat's stable cores/hatarib_libretro.dll.zip (2026-04-04), the 0.3 tag" + source_ref: "core/core.c:963-976, core/core_config.c:961-969, 985, 1286-1318, 1408-1413, hatari/src/main.c:898, 921-925, 1019" + keys: + - key: "hatarib_tos" + source_ref: "core/core_config.c:1082-1104, core/core_file.c:256-259, 763-769, hatari/src/tos.c:996-1056" + condition: " is offered only when system/tos.img exists, a hatarib/ value only for a file with an img, rom, bin or tos extension (core/core_config.c:1293-1318, core/core_file.c:707-716)" + notes: | hatariB carries Hatari 2.6.1 in its hatari/ directory, every change fenced by __LIBRETRO__ (DEVELOP.md:7-14, DEVELOP.md:44-48), and diff --git a/emulators/kronos.yml b/emulators/kronos.yml index 16ed978ee..8cef01caf 100644 --- a/emulators/kronos.yml +++ b/emulators/kronos.yml @@ -20,6 +20,19 @@ systems: - sega-saturn - sega-stv +config_reads: + - file: "kronos.ini" + platforms: [retrobat] + revision: "6750417f3d6fef1b7be9adf52e707cc78aa41ee0" + revision_from: "Kronos 2.7.0 without a commit id in kronos.exe of RetroBat's stable kronos.7z, linked 2024-12-31: the last FCare/Kronos commit before that date, the cited files unchanged from aca9865c to the 2.7.0_official_release tag 58352d6d" + source_ref: "upstream yabause/src/port/qt/Settings.cpp:50-61, 95-103, upstream yabause/src/port/qt/Settings.h:25-26, upstream yabause/src/CMakeLists.txt:763, upstream yabause/src/port/qt/main.cpp:55-57" + keys: + - key: "[1.0] General\\EnableEmulatedBios" + source_ref: "upstream yabause/src/port/qt/YabauseThread.cpp:392-393" + - key: "[1.0] General\\Bios" + source_ref: "upstream yabause/src/port/qt/YabauseThread.cpp:394-395, upstream yabause/src/port/qt/VolatileSettings.cpp:20-27, upstream yabause/src/ctrl/src/yabause.c:497-509, 575-587, 626-630, upstream yabause/src/sys/memory/src/memory.c:1343-1346" + condition: "General\\EnableEmulatedBios false; an empty path runs the HLE BIOS, a path that fails to load stops init" + notes: | Kronos is a Sega Saturn and ST-V arcade emulator by FCare, forked from Yabause. Requires OpenGL 4.3 compute shaders. diff --git a/emulators/ssf.yml b/emulators/ssf.yml index 45f70463b..5424a8a0c 100644 --- a/emulators/ssf.yml +++ b/emulators/ssf.yml @@ -21,6 +21,17 @@ systems: - sega-stv mode: standalone +config_reads: + - file: "SSF.ini" + platforms: [retrobat] + binary_sha256: "00f43e448592e5e8359eba7da4de5b3a16e6840ad8faf9cf182f738c4e25293a" + revision_from: "SSF PreviewVer R28 (string at 0x1a46634): the 32-bit SSF.exe of RetroBat's stable ssf.7z (2026-04-04), byte-identical to the one in SSF_PreviewVer_R28.zip of release tag PreviewVer" + source_ref: "SSF.exe 0x5b0b41-0x5b0b81, SSF.exe 0x5b9785, SSF.exe 0x5ac8cc-0x5ac8e2, SSF.exe 0x5ac94c, SSF.exe 0x5953d0, SSF.exe 0x5954c3-0x5954db, SSF.exe 0xb35e20" + keys: + - key: "[Peripheral] SaturnBIOS" + source_ref: "SSF.exe 0x52bc85-0x52bcd8, SSF.exe 0x5ac9e4, SSF.exe 0x5aab87-0x5aab93, SSF.exe 0x5b2905-0x5b2936, SSF.exe 0xb36022-0xb360cf" + condition: "[Program4] NoBIOS off (0 in the embedded template) and Ctrl not held at launch, in Saturn mode (no left Shift, no zip); an empty path or a read short of 0x80000 bytes boots without BIOS" + notes: | Sega Saturn and ST-V emulator by Shima, closed source and distributed as a binary. Two artifacts are read here and they describe the same emulator: the diff --git a/emulators/yabasanshiro.yml b/emulators/yabasanshiro.yml index 3afe0efdc..513e95600 100644 --- a/emulators/yabasanshiro.yml +++ b/emulators/yabasanshiro.yml @@ -39,6 +39,19 @@ data_root: source_ref: "yabause/src/retro_arena/main.cpp:292-301" note: "Created at startup. Holds backup.bin, keymapv2.json, the per-game settings files, save states and screenshots. No firmware is read from it." +config_reads: + - file: "yabause.ini" + platforms: [retrobat] + binary_sha256: "54f89ad3d45f078422616b794e9f7d400a37af0fbb00b31d3ba5a3b66516bf27" + revision_from: "yabasanshiro.exe of RetroBat's stable yabasanshiro.7z (2026-08-08), version 1.20.32 (0x141206de8), the yabasanshiro-1.20.32-3989ac release of 2026-07-31 whose source is not public" + source_ref: "yabasanshiro.exe 0x14001ae2c-0x14001ae58, yabasanshiro.exe 0x1400cea2c-0x1400cea33, yabasanshiro.exe 0x1400ce4d5-0x1400ce59e, yabasanshiro.exe 0x1400ce5a3-0x1400ce61f, yabasanshiro.exe 0x1400ce44a-0x1400ce496, yabasanshiro.exe 0x1400ce339-0x1400ce394" + keys: + - key: "[0.9.11] General\\EnableEmulatedBios" + source_ref: "yabasanshiro.exe 0x1400d248f-0x1400d2512" + - key: "[0.9.11] General\\Bios" + source_ref: "yabasanshiro.exe 0x1400d251b-0x1400d258c, yabasanshiro.exe 0x1400cf162-0x1400cf166, yabasanshiro.exe 0x14016d957-0x14016d98f, yabasanshiro.exe 0x14016f3ad-0x14016f4b4" + condition: "General\\EnableEmulatedBios false; an empty value runs the HLE BIOS (0x14016d997-0x14016d9f7), a file that does not open stops init" + notes: | YabaSanshiro is a Sega Saturn emulator by devmiyax, forked from Yabause. Requires OpenGL 3.3 or OpenGL ES 3.0. diff --git a/emulators/ymir.yml b/emulators/ymir.yml index f1a2dad6a..07784acf8 100644 --- a/emulators/ymir.yml +++ b/emulators/ymir.yml @@ -42,6 +42,22 @@ data_root: source_ref: "apps/ymir-sdl3/src/app/profile.cpp:29-34, apps/ymir-sdl3/src/app/app.cpp:342-437, apps/ymir-sdl3/CMakeLists.txt:647-648" note: "Portable profile: the current directory, then the folder of the executable, when it holds Ymir.toml. The first launch asks between the two. -p names a folder, -u forces this one. Always this folder under Flatpak" +config_reads: + - file: "Ymir.toml" + platforms: [retrobat] + revision: "54fead6a0001d3e4b6741a8d095ee8342266c3dc" + revision_from: "version 0.3.3 of ymir-sdl3.exe in RetroBat's stable ymir.7z (2026-08-08), byte-identical to the v0.3.3 windows-x86_64-AVX2 release asset" + source_ref: "apps/ymir-sdl3/src/app/settings.hpp:40, apps/ymir-sdl3/src/app/app.cpp:340-359, 412, apps/ymir-sdl3/src/app/profile.cpp:36-62" + keys: + - key: + - "[System.IPL] Override" + - "[System.IPL] Path" + source_ref: "apps/ymir-sdl3/src/app/settings.cpp:1236-1240, 2805-2813, apps/ymir-sdl3/src/app/services/rom_service.cpp:109-114, apps/ymir-sdl3/src/util/rom_loader.cpp:9-22, apps/ymir-sdl3/src/app/app.cpp:451-462" + condition: "Override true and Path not empty: the path is opened without an existence check and must be 512 KiB" + - key: "[General.PathOverrides] IPLROMImages" + source_ref: "apps/ymir-sdl3/src/app/settings.cpp:1190-1197, apps/ymir-sdl3/src/app/profile.cpp:94-104, apps/ymir-sdl3/src/app/services/rom_service.cpp:67-73, 116-175, apps/ymir-sdl3/src/app/rom_manager.cpp:14-45" + condition: "Override false or Path empty: the folder is scanned recursively for 512 KiB files known by XXH128" + notes: | Ymir is a Sega Saturn emulator by StrikerX3. The libretro port by warmenhoven wraps the shared ymir-core library on the libretro branch of diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 7bb563009..2e11e9d4e 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -1165,12 +1165,14 @@ "type": "object", "additionalProperties": false, "required": ["file", "source_ref", "keys"], - "dependentRequired": {"revision": ["platforms", "revision_from"]}, + "dependentRequired": {"revision": ["platforms", "revision_from"], "binary_sha256": ["platforms", "revision_from"]}, + "not": {"required": ["revision", "binary_sha256"]}, "properties": { "file": {"type": "string", "minLength": 1, "description": "The config as the code finds it: a name, a glob for any file a flag passes (*.uae), a path, the flag itself, \"command line\" or \"core options\""}, "platforms": {"type": "array", "items": {"type": "string"}, "minItems": 1, "uniqueItems": true, "description": "Registry slugs of the platforms shipping the build these refs are read at, each with a pin_build entry unless revision names the build"}, "revision": {"type": "string", "pattern": "^[0-9a-f]{40}$", "description": "The commit these refs are read at, for platforms installing a package no build file of theirs pins (RetroBat's stable archives, a Flathub app)"}, - "revision_from": {"type": "string", "minLength": 1, "description": "Where revision was read: the version string of the shipped binary, the Flathub manifest commit"}, + "binary_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The shipped binary these refs are read in, for a build whose source is not public (SSF, YabaSanshiro 1.20): its refs are virtual addresses of that file"}, + "revision_from": {"type": "string", "minLength": 1, "description": "Where revision or binary_sha256 was read: the version string of the shipped binary, the Flathub manifest commit, the archive the binary comes from"}, "source_ref": {"type": "string", "minLength": 1}, "keys": { "type": "array", diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 94c663b81..99abe41d4 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -1652,7 +1652,10 @@ def _check_build_refs( results: list[dict] = [] for citation, read in deferred: for platform in read.get("platforms") or []: - revision = str(read.get("revision") or "") or build_revision(profile, str(platform)) + # A closed build is named by its binary: an address ref is + # skipped below, a source path cannot exist at a sha256. + revision = (str(read.get("revision") or read.get("binary_sha256") or "") + or build_revision(profile, str(platform))) row = {"platform": str(platform), "revision": revision or "", "ref": citation.ref, "field": citation.field} if revision is None: diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index dc84746b6..055495866 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -147,7 +147,7 @@ def _config_reads_without_build(path: Path, document: object) -> list[str]: for index, read in enumerate(document.get("config_reads") or []): if not isinstance(read, dict): continue - if read.get("revision"): + if read.get("revision") or read.get("binary_sha256"): continue for platform in read.get("platforms") or []: if str(platform) not in builds: diff --git a/tests/test_launcher_evidence.py b/tests/test_launcher_evidence.py index 5f01c2929..31d7d549f 100644 --- a/tests/test_launcher_evidence.py +++ b/tests/test_launcher_evidence.py @@ -261,6 +261,8 @@ class Invariants(unittest.TestCase): ), []) pinned = [{**reads[0], "revision": "c" * 40, "revision_from": "the exe"}] self.assertEqual(_config_reads_without_build(path, {"config_reads": pinned}), []) + closed = [{**reads[0], "binary_sha256": "d" * 64, "revision_from": "the archive"}] + self.assertEqual(_config_reads_without_build(path, {"config_reads": closed}), []) class BuildRefs(unittest.TestCase): @@ -299,6 +301,37 @@ class BuildRefs(unittest.TestCase): self.assertNotIn(("b" * 40, "ROCKNIX x.patch"), asked) + def test_a_closed_build_is_read_in_its_binary(self) -> None: + """An address ref is read in the binary binary_sha256 names, never + fetched; a source path cannot exist at a closed build.""" + read = { + "file": "SSF.ini", "platforms": ["retrobat"], "binary_sha256": "e" * 64, + "revision_from": "the archive", "source_ref": "SSF.exe 0x10-0x20", "keys": [], + } + + class View: + repo, field = "repo", "source" + + class Citation: + def __init__(self, ref: str) -> None: + self.ref, self.field = ref, "c" + + asked: list[str] = [] + + def fetch(_repo: str, _revision: str, path: str) -> list[str] | None: + asked.append(path) + return None + + rows = _check_build_refs( + {"config_reads": [read]}, + [(Citation("SSF.exe 0x10-0x20"), read), (Citation("src/a.c:3"), read)], + [View()], fetch, + ) + self.assertEqual([r["status"] for r in rows], ["BUILD_OK", "BUILD_GONE"]) + self.assertEqual({r["revision"] for r in rows}, {"e" * 64}) + self.assertEqual(asked, ["src/a.c"]) + + class RepositoryRecords(unittest.TestCase): """The placement chains the audit of 2026-11-10 found moving files.""" From 8770b9c067bb09de4dfcc185dd78f7e0684d3e14 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:57:12 +0100 Subject: [PATCH 3/6] fix: install emuelec configs from the boot copy duckstation.sh quotes its glob, so its copy names a file called * and installs nothing. userconfig-setup copies /usr/config into /storage/.config at every boot, an existing file kept (false | cp -i), and /emuelec links there: that copy installs settings.ini and advmame.rc. A quoted glob is no install, and a redirection ends the copy's operands. Rescraped at v4.8: only these installs change. --- platforms/emuelec.yml | 67 ++++++++++++++++++---------- scripts/scraper/emuelec_launchers.py | 41 +++++++++++++++-- scripts/scraper/shell_launchers.py | 21 ++++++++- tests/test_launchers_emuelec.py | 47 +++++++++++++++++-- 4 files changed, 144 insertions(+), 32 deletions(-) diff --git a/platforms/emuelec.yml b/platforms/emuelec.yml index 3c181e6a4..01167303e 100644 --- a/platforms/emuelec.yml +++ b/platforms/emuelec.yml @@ -4295,8 +4295,8 @@ launchers: \ PathPAL unset" fails_when_missing: false installs: - path: /emuelec/configs/duckstation/settings.ini - source_ref: "packages/sx05re/emulators/duckstation/scripts/duckstation.sh:15" + path: /storage/.config/emuelec/configs/duckstation/settings.ini + source_ref: "packages/sx05re/emulators/duckstation/package.mk:56, packages/sysutils/systemd/scripts/userconfig-setup:12" kept_if_present: true overrides: [] source_ref: "packages/sx05re/emuelec/bin/emuelecRunEmu.sh:207-209, packages/sx05re/emulators/duckstation/config/settings.ini:6-9,\ @@ -4372,7 +4372,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4391,7 +4392,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4409,7 +4411,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4428,7 +4431,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4446,7 +4450,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4465,7 +4470,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4483,7 +4489,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4502,7 +4509,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4520,7 +4528,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4539,7 +4548,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4557,7 +4567,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4576,7 +4587,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4594,7 +4606,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4613,7 +4626,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4631,7 +4645,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4650,7 +4665,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4667,7 +4683,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4684,7 +4701,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4703,7 +4721,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4721,7 +4740,8 @@ launchers: depends_on_option: system != arcade installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" @@ -4740,7 +4760,8 @@ launchers: fails_when_missing: false installs: path: /emuelec/configs/advmame/advmame.rc - source_ref: "packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" + source_ref: "packages/sx05re/emulators/advancemame/package.mk:45-49, packages/sysutils/systemd/scripts/userconfig-setup:12,\ + \ packages/sx05re/emulators/advancemame/bin/advmame.sh:17, 21" kept_if_present: false overrides: - "packages/sx05re/emulators/advancemame/bin/advmame.sh:27" diff --git a/scripts/scraper/emuelec_launchers.py b/scripts/scraper/emuelec_launchers.py index e52a23fc9..98388983e 100644 --- a/scripts/scraper/emuelec_launchers.py +++ b/scripts/scraper/emuelec_launchers.py @@ -40,6 +40,10 @@ EMULATORS = f"{SX05RE}/emulators" LIBRETRO = f"{SX05RE}/libretro" RETROARCH_MK = f"{SX05RE}/libretro_base/retroarch/package.mk" BIOS_ROOT = "/storage/roms/bios" +EMUELEC_MK = f"{SX05RE}/emuelec/package.mk" +# The copy of every package's /usr/config into /storage/.config at each boot, +# an existing file kept (cp -i answered by false). +USERCONFIG = "packages/sysutils/systemd/scripts/userconfig-setup" # The roms root that holds it (packages/sx05re/libretro_base/retroarch/package.mk:112,117). ROMS_ROOT = "/storage/roms" @@ -106,6 +110,19 @@ class Runs: return (RUN_EMU, set(self.of(name).lines)) +def _storage_path(tree: RepoTree, target: str) -> str: + """target with the /emuelec link resolved to the folder it points at. + + The emuelec package links /emuelec to /storage/.config/emuelec, where + the first-boot copy of /usr/config lands. + """ + mk = _read(tree, EMUELEC_MK) + _line, link = mk.find(r"ln -sf (/storage/\S+) \$\{INSTALL\}(/emuelec)$", "/emuelec link") + if target == link.group(2) or target.startswith(link.group(2) + "/"): + return link.group(1) + target[len(link.group(2)):] + return target + + def _evidence( record: Launcher, script: Script, @@ -113,15 +130,28 @@ def _evidence( keys: list[str], source_name: str = "", rewrites: Iterable[int] = (), + tree: RepoTree | None = None, + installed: Iterable[tuple[str, Iterable[int]]] = (), ) -> Launcher: - """A config record with the start script lines that put its file in place. + """A config record with the lines that put its file in place: the + first-boot copy of /usr/config when ``tree`` is given, then the start + script's own. ``installed`` are the package.mk lines putting the file + in /usr/config. ``rewrites`` are lines the extractor read as rewriting the record's values without naming their keys (advmame.sh's sed of /roms/mame). """ - placement = config_placement(script, target, keys, source_name=source_name) + placements = [] + if tree is not None: + setup = _read(tree, USERCONFIG) + placements.append(config_placement( + setup, _storage_path(tree, target), required=False, sources=("/usr/config/",) + )) + placement = config_placement(script, target, keys, source_name=source_name, + required=not placements or not placements[0].install) placement.overrides |= set(rewrites) - fields = evidence_fields([placement]) + placements.append(placement) + fields = evidence_fields(placements, installed) record.installs = fields["installs"] record.kept_if_present = fields["kept_if_present"] record.overrides = fields["overrides"] @@ -356,6 +386,8 @@ def duckstation(tree: RepoTree, runs: Runs) -> list[Launcher]: script, script.expand("${CONFIG_DIR}/settings.ini", run_line, set()), [f"[{s}] {k}" for s, k in (folder_key, *file_keys)], + tree=tree, + installed=[(mk.path, {mk_line})], )] @@ -569,7 +601,8 @@ def advancemame(tree: RepoTree, runs: Runs) -> list[Launcher]: target = script.expand("${CONFIG_DIR}/advmame.rc", run_line, set()) def proved(record: Launcher, key: str) -> Launcher: - return _evidence(record, script, target, [key], rewrites={sed_line}) + return _evidence(record, script, target, [key], rewrites={sed_line}, tree=tree, + installed=[(mk.path, {mk_line for _, _, mk_line in configs})]) records = [] for key, folder in value: diff --git a/scripts/scraper/shell_launchers.py b/scripts/scraper/shell_launchers.py index d847b2666..e71936986 100644 --- a/scripts/scraper/shell_launchers.py +++ b/scripts/scraper/shell_launchers.py @@ -483,6 +483,15 @@ def _guarded(script: Script, number: int, target: str) -> bool: return False +def _quoted_glob(command: str) -> bool: + """Whether the source of a copy holds a "*" only inside quotes.""" + found = [w for w in words(command)[1:] if not w.raw.startswith("-")] + if len(found) < 2: + return False + raw = found[0].raw + return "*" in raw and "*" not in re.sub(r'"[^"]*"|\'[^\']*\'', "", raw) + + def config_placement( script: Script, target: str, @@ -511,12 +520,16 @@ def config_placement( args = [] for word in words(line[copy.start(1):])[1:]: # The command ends at a separator or a redirection. - if word.raw in {";", "&&", "||", "|"} or re.match(r"\d*>", word.raw): + if word.raw in {";", "&&", "||", "|"} or re.match(r"\d*>|&>", word.raw): break args.append(word.text) operands = [a for a in args if not a.startswith("-")] if not operands: continue + # A quoted "*" is a file named *, which no package installs: + # the copy does nothing (EmuELEC's duckstation.sh). + if _quoted_glob(line[copy.start(1):]): + continue destination = script.expand(operands[-1], number, set()).rstrip("/") source = script.expand(operands[0], number, set()).rstrip("/") hits = target == destination or target.startswith(destination + "/") or ( @@ -536,7 +549,11 @@ def config_placement( continue if sources and not source.startswith(sources): continue - skip = "-n" in args or "--ignore-existing" in args + # cp -i fed by false answers no to every overwrite. + prompted = any( + a.startswith("-") and not a.startswith("--") and "i" in a[1:] for a in args + ) and re.search(r"\bfalse\s*\|\s*$", line[:copy.start(1)]) + skip = "-n" in args or "--ignore-existing" in args or bool(prompted) else: skip = False install.add(number) diff --git a/tests/test_launchers_emuelec.py b/tests/test_launchers_emuelec.py index 630547701..8ab6fa3fe 100644 --- a/tests/test_launchers_emuelec.py +++ b/tests/test_launchers_emuelec.py @@ -229,11 +229,25 @@ DUCKSTATION_SH = text( CONFIG_DIR="/emuelec/configs/duckstation" duckstation-nogui -fullscreen -settings "${CONFIG_DIR}/settings.ini" -- "${1}" if [ ! -d "${CONFIG_DIR}" ]; then - cp -rf /usr/config/emuelec/configs/duckstation/* "${CONFIG_DIR}" + cp -rf "/usr/config/emuelec/configs/duckstation/*" "${CONFIG_DIR}" fi """ ) +# packages/sysutils/systemd/scripts/userconfig-setup:11-12 at v4.8. +USERCONFIG = text( + """ + # Copy config files, but don't overwrite - this should replace our sample files + false | cp -iRp /usr/config/* /storage/.config/ &>/dev/null +""" +) + +EMUELEC_MK = text( + """ + ln -sf /storage/.config/emuelec ${INSTALL}/emuelec +""" +) + def yabasanshiro_sh(program: str) -> str: return text( @@ -400,6 +414,8 @@ FILES = { f"{EMU}/duckstation/config/settings.ini": DUCKSTATION_INI, f"{EMU}/duckstation/package.mk": DUCKSTATION_MK, f"{EMU}/duckstation/scripts/duckstation.sh": DUCKSTATION_SH, + el.USERCONFIG: USERCONFIG, + el.EMUELEC_MK: EMUELEC_MK, el.YABASANSHIRO_SCRIPTS[0]: yabasanshiro_sh("yabasanshiro"), el.YABASANSHIRO_SCRIPTS[1]: yabasanshiro_sh("yabasanshiro1_5"), f"{EMU}/jzintv/scripts/jzintv.sh": JZINTV_SH, @@ -540,12 +556,37 @@ class StandaloneRecords(unittest.TestCase): (record,) = records(el.duckstation) self.assertEqual((record["kind"], record["search_dir"]), ("scans_dir", "/storage/roms/bios")) - def test_duckstation_config_is_kept_when_present(self): + def test_duckstation_config_comes_from_the_boot_copy(self): + """duckstation.sh quotes its glob: its copy names a file called *, so + the settings.ini in place is the one userconfig-setup copies from + /usr/config, an existing file kept (false | cp -i).""" (record,) = records(el.duckstation) - self.assertEqual(record["installs"]["path"], "/emuelec/configs/duckstation/settings.ini") + self.assertEqual( + record["installs"], + {"path": "/storage/.config/emuelec/configs/duckstation/settings.ini", + "source_ref": f"{EMU}/duckstation/package.mk:1, {el.USERCONFIG}:2"}, + ) self.assertIs(record["kept_if_present"], True) self.assertEqual(record["overrides"], []) + def test_an_unquoted_glob_in_the_start_script_installs_too(self): + unquoted = DUCKSTATION_SH.replace( + '"/usr/config/emuelec/configs/duckstation/*"', "/usr/config/emuelec/configs/duckstation/*" + ) + (record,) = records(el.duckstation, **{f"{EMU}/duckstation/scripts/duckstation.sh": unquoted}) + self.assertEqual(record["installs"]["path"], "/emuelec/configs/duckstation/settings.ini") + self.assertIs(record["kept_if_present"], True) + + def test_without_a_boot_copy_the_quoted_glob_installs_nothing(self): + with self.assertRaises(LauncherPatternMissing): + records(el.duckstation, **{el.USERCONFIG: "#!/bin/sh\n"}) + + def test_an_overwriting_boot_copy_does_not_keep_the_file(self): + (record,) = records(el.duckstation, **{ + el.USERCONFIG: "cp -Rp /usr/config/* /storage/.config/ &>/dev/null\n", + }) + self.assertIs(record["kept_if_present"], False) + def test_duckstation_with_a_fixed_bios_path_raises(self): changed = DUCKSTATION_INI.replace("PathPAL =", "PathPAL = /storage/roms/bios/scph5502.bin") with self.assertRaises(LauncherPatternMissing): From b8ed9b735aedc3956e34d2cfee85c8040cfa7373 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:06:28 +0100 Subject: [PATCH 4/6] feat: profile the retroarch frontend config reads AmberELEC and ROCKNIX install a retroarch.cfg whose system_directory points every core at the BIOS folder. retroarch.yml (type utility, no file, no system) reads that key, and ROCKNIX's systemfiles_in_content_dir, at the RetroArch each one compiles: v1.22.2 and bdba046f. all_libretro never picks a utility profile, and lists naming retroarch gain no file. A libretro core option a frontend record writes stays the core's to read: RetroDECK's pcsx2_bios keeps lrps2 as its reader. --- emulators/retroarch.yml | 58 +++++++++++++++++++++++++++++++++ scripts/launchers.py | 12 +++++-- tests/test_launcher_evidence.py | 28 ++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 emulators/retroarch.yml diff --git a/emulators/retroarch.yml b/emulators/retroarch.yml new file mode 100644 index 000000000..6a04ef897 --- /dev/null +++ b/emulators/retroarch.yml @@ -0,0 +1,58 @@ +emulator: "RetroArch" +type: utility +core_classification: other +source: "https://github.com/libretro/RetroArch" +upstream: "https://github.com/libretro/RetroArch" +profiled_date: "2026-11-11" +source_commit: "69a4f0ea1e8aaf442ae4858f2e7f2b31a1776576" +core_version: "v1.22.2" +version_basis: + kind: tag + tag: "v1.22.2" + distance: 0 +display_name: "RetroArch (libretro frontend)" +cores: + - retroarch +systems: [] + +pin_build: + - platform: amberelec + repo: "https://github.com/AmberELEC/AmberELEC" + revision: "a21975bfb1a3af68f66e07a931b994b523e1e6e2" + path: "packages/games/tools/retroarch/package.mk" + variable: PKG_VERSION + value: "69a4f0ea1e8aaf442ae4858f2e7f2b31a1776576" + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261101" + path: "projects/ROCKNIX/packages/emulators/libretro/retroarch/package.mk" + variable: PKG_VERSION + value: "bdba046fa6766380bc2457532f38e589df769aaf" +config_reads: + - file: "retroarch.cfg" + platforms: [amberelec] + source_ref: "retroarch.c:6968-6969, 7148-7155, 7244, configuration.c:3469, 3779, 3824, 3863-3884, 5014-5018, libretro-common/file/config_file.c:840" + keys: + - key: "system_directory" + source_ref: "configuration.c:1645, 4063-4069, 4110-4115, 4346-4347, libretro-common/file/config_file.c:1180-1189, libretro-common/file/file_path.c:1060-1098, runloop.c:1935-1976" + condition: "systemfiles_in_content_dir false (its default, config.def.h:925); LIBRETRO_SYSTEM_DIRECTORY in the environment replaces the value, an empty value hands the content's folder" + - file: "retroarch.cfg" + platforms: [rocknix] + source_ref: "retroarch.c:7272-7273, 7452-7459, 7548, configuration.c:3565, 3886, 3931, 3970-4010, 5174-5176, libretro-common/file/config_file.c:817" + keys: + - key: "system_directory" + source_ref: "configuration.c:1688, 4193-4199, 4240-4245, 4494-4495, libretro-common/file/config_file.c:1190-1199, libretro-common/file/file_path.c:1066-1100, runloop.c:1949-1990" + condition: "systemfiles_in_content_dir false; LIBRETRO_SYSTEM_DIRECTORY in the environment replaces the value, an empty value hands the content's folder" + - key: "systemfiles_in_content_dir" + source_ref: "configuration.c:1870, runloop.c:1949-1990" + condition: "true hands the content's folder in place of system_directory" + +notes: | + The libretro frontend. It loads each core and answers + RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY with the system_directory of the + config it was started with (-c, else the default path), a leading ~ + expanded to $HOME. The cores read their BIOS files below that folder, so + their profiles hold the files and this one holds none. + ref: retroarch.c:7148-7155, configuration.c:3824, runloop.c:1935-1976 + +files: [] diff --git a/scripts/launchers.py b/scripts/launchers.py index c790af076..138014337 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -468,7 +468,12 @@ def config_evidence( readers: list[str] = [] uncovered = list(keys) loaded = False - for key in _named_profiles(record, profiles, platform) or sorted(profiles): + named = _named_profiles(record, profiles, platform) + if file == CORE_OPTIONS: + # A core option is read by the core RetroArch runs, not by the + # frontend a record names (RetroDECK's retroarch component). + named = [k for k in named if "libretro" in str(profiles[k].get("type") or "")] + for key in named or sorted(profiles): profile = profiles[key] handles = _handle_refs(profile) for read in profile.get("config_reads") or []: @@ -1319,7 +1324,10 @@ class LauncherLayer: for key in self._profiles_named(emulator, profiles, relevant) } # RetroDECK names its frontend, retroarch, and the core option it - # writes, pcsx2_bios, names the core. + # writes, pcsx2_bios, names the core: a libretro core option + # belongs to a libretro core. + if place.build == "libretro" and place.kind == "core_option": + owners = {k for k in owners if "libretro" in str(profiles[k].get("type") or "")} if not owners: owners = set(setting_readers(place.passes_to, profiles, relevant)) for key in sorted(owners): diff --git a/tests/test_launcher_evidence.py b/tests/test_launcher_evidence.py index 31d7d549f..bce6fe7e7 100644 --- a/tests/test_launcher_evidence.py +++ b/tests/test_launcher_evidence.py @@ -265,6 +265,34 @@ class Invariants(unittest.TestCase): self.assertEqual(_config_reads_without_build(path, {"config_reads": closed}), []) +class CoreOptions(unittest.TestCase): + def test_a_core_option_is_read_by_the_core_not_the_frontend(self) -> None: + """RetroDECK's retroarch component writes pcsx2_bios: the frontend + names the record, the core RetroArch runs reads the option.""" + profiles = { + "retroarch": {"emulator": "RetroArch", "type": "utility", "cores": ["retroarch"], + "config_reads": [{"file": "retroarch.cfg", "source_ref": "a.c:1", + "keys": [{"key": "system_directory", + "source_ref": "a.c:2"}]}]}, + "lrps2": {"emulator": "LRPS2", "type": "libretro", "cores": ["lrps2"], + "config_reads": [{"file": "core options", "source_ref": "main.cpp:1", + "keys": [{"key": "pcsx2_bios", + "source_ref": "main.cpp:2"}]}]}, + } + option = { + "emulator": "retroarch", "build": "libretro", "kind": "core_option", + "names": ["scph10000.bin"], "passes_to": "core option pcsx2_bios", + "installs": {"path": "$core_options", "source_ref": "prepare.sh:24"}, + "overrides": [], "source_ref": "rd_config/retroarch-core-options.cfg:515", + } + evidence = config_evidence(option, profiles, "retrodeck") + self.assertTrue(evidence.proven, evidence.missing) + self.assertEqual(evidence.readers, ("lrps2",)) + system = {**option, "kind": "passes_path", "passes_to": "retroarch.cfg system_directory", + "source_ref": "rd_config/retroarch.cfg:44"} + self.assertEqual(config_evidence(system, profiles, "retrodeck").readers, ("retroarch",)) + + class BuildRefs(unittest.TestCase): def test_reads_for_a_shipped_build_are_read_at_that_build(self) -> None: document = { From 87683d9bd7eb66c6e3312fd88cc5f60410a5d4c4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:17:04 +0100 Subject: [PATCH 5/6] refactor: keep build refs out of conditions A line cited in a config_reads condition is read as prose at the profile's pin, not at the build the entry describes. The lines move into the key's source_ref, which is read at that build. --- emulators/hatari2014.yml | 4 ++-- emulators/hatarib.yml | 4 ++-- emulators/retroarch.yml | 4 ++-- emulators/yabasanshiro.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/emulators/hatari2014.yml b/emulators/hatari2014.yml index ff7f84be6..7970e7a32 100644 --- a/emulators/hatari2014.yml +++ b/emulators/hatari2014.yml @@ -30,8 +30,8 @@ config_reads: source_ref: "source libretro/libretro.c:264-300, 372, 393-413, 437-440, 984-1001, 1418, source libretro/libretro_core_options.h:129-145" keys: - key: "hatari_tosimage" - source_ref: "source libretro/libretro.c:617, 639-649, 1612-1617, source src/main.c:913-917, source src/tos.c:575-590" - condition: "a value other than default is a file of system/hatari/tos ending in img (libretro/libretro.c:288-299)" + source_ref: "source libretro/libretro.c:288-299, 617, 639-649, 1612-1617, source src/main.c:913-917, source src/tos.c:575-590" + condition: "a value other than default is a file of system/hatari/tos ending in img" notes: | Hatari 2014 is the libretro port of Hatari 1.8.0 (src/includes/version.h:16), diff --git a/emulators/hatarib.yml b/emulators/hatarib.yml index 97b3fa8b9..f2bf7913f 100644 --- a/emulators/hatarib.yml +++ b/emulators/hatarib.yml @@ -28,8 +28,8 @@ config_reads: source_ref: "core/core.c:963-976, core/core_config.c:961-969, 985, 1286-1318, 1408-1413, hatari/src/main.c:898, 921-925, 1019" keys: - key: "hatarib_tos" - source_ref: "core/core_config.c:1082-1104, core/core_file.c:256-259, 763-769, hatari/src/tos.c:996-1056" - condition: " is offered only when system/tos.img exists, a hatarib/ value only for a file with an img, rom, bin or tos extension (core/core_config.c:1293-1318, core/core_file.c:707-716)" + source_ref: "core/core_config.c:1082-1104, 1293-1318, core/core_file.c:256-259, 707-716, 763-769, hatari/src/tos.c:996-1056" + condition: " is offered only when system/tos.img exists, a hatarib/ value only for a file with an img, rom, bin or tos extension" notes: | hatariB carries Hatari 2.6.1 in its hatari/ directory, every change diff --git a/emulators/retroarch.yml b/emulators/retroarch.yml index 6a04ef897..e5feda3ef 100644 --- a/emulators/retroarch.yml +++ b/emulators/retroarch.yml @@ -34,8 +34,8 @@ config_reads: source_ref: "retroarch.c:6968-6969, 7148-7155, 7244, configuration.c:3469, 3779, 3824, 3863-3884, 5014-5018, libretro-common/file/config_file.c:840" keys: - key: "system_directory" - source_ref: "configuration.c:1645, 4063-4069, 4110-4115, 4346-4347, libretro-common/file/config_file.c:1180-1189, libretro-common/file/file_path.c:1060-1098, runloop.c:1935-1976" - condition: "systemfiles_in_content_dir false (its default, config.def.h:925); LIBRETRO_SYSTEM_DIRECTORY in the environment replaces the value, an empty value hands the content's folder" + source_ref: "configuration.c:1645, 4063-4069, 4110-4115, 4346-4347, libretro-common/file/config_file.c:1180-1189, libretro-common/file/file_path.c:1060-1098, runloop.c:1935-1976, config.def.h:925" + condition: "systemfiles_in_content_dir false, its default; LIBRETRO_SYSTEM_DIRECTORY in the environment replaces the value, an empty value hands the content's folder" - file: "retroarch.cfg" platforms: [rocknix] source_ref: "retroarch.c:7272-7273, 7452-7459, 7548, configuration.c:3565, 3886, 3931, 3970-4010, 5174-5176, libretro-common/file/config_file.c:817" diff --git a/emulators/yabasanshiro.yml b/emulators/yabasanshiro.yml index 513e95600..cfe8bc346 100644 --- a/emulators/yabasanshiro.yml +++ b/emulators/yabasanshiro.yml @@ -49,8 +49,8 @@ config_reads: - key: "[0.9.11] General\\EnableEmulatedBios" source_ref: "yabasanshiro.exe 0x1400d248f-0x1400d2512" - key: "[0.9.11] General\\Bios" - source_ref: "yabasanshiro.exe 0x1400d251b-0x1400d258c, yabasanshiro.exe 0x1400cf162-0x1400cf166, yabasanshiro.exe 0x14016d957-0x14016d98f, yabasanshiro.exe 0x14016f3ad-0x14016f4b4" - condition: "General\\EnableEmulatedBios false; an empty value runs the HLE BIOS (0x14016d997-0x14016d9f7), a file that does not open stops init" + source_ref: "yabasanshiro.exe 0x1400d251b-0x1400d258c, yabasanshiro.exe 0x1400cf162-0x1400cf166, yabasanshiro.exe 0x14016d957-0x14016d98f, yabasanshiro.exe 0x14016f3ad-0x14016f4b4, yabasanshiro.exe 0x14016d997-0x14016d9f7" + condition: "General\\EnableEmulatedBios false; an empty value runs the HLE BIOS, a file that does not open stops init" notes: | YabaSanshiro is a Sega Saturn emulator by devmiyax, forked from Yabause. From 0a6c70c388e94192f40e7ba723bad0558e44c8c3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:17:04 +0100 Subject: [PATCH 6/6] feat: read retrodeck and emudeck at shipped builds RetroDECK carries GZDoom g4.14.2 in its components and installs LRPS2 14d19f8 from the buildbot's stable 1.22.2 cores. EmuDeck runs the Flathub ares, v148. Each gets its own config_reads entry, and an entry read at the pin no longer stands for a platform another entry names. At v148 ares reads NeoGeoAES/Firmware/BIOS.World, so that EmuDeck record holds. It names MSX2 firmware MAIN and SUB, and PC Engine CD and SuperGrafx CD firmware by card: EmuDeck's BIOS.* keys for those three are never read and stay unproven. --- emulators/ares.yml | 47 +++++++++++++++++++++++++++++++++ emulators/gzdoom.yml | 12 +++++++++ emulators/lrps2.yml | 9 +++++++ scripts/launchers.py | 10 ++++--- tests/test_launcher_evidence.py | 17 ++++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/emulators/ares.yml b/emulators/ares.yml index 35623cf00..fe77a7e21 100644 --- a/emulators/ares.yml +++ b/emulators/ares.yml @@ -98,6 +98,53 @@ config_reads: - "PlayStation/Firmware/BIOS.Europe" source_ref: "desktop-ui/emulator/playstation.cpp:19-23, 87-101" condition: "the disc's region" + - file: "settings.bml" + platforms: [emudeck] + revision: "0aafd85789215e84e1e43415c07d4c88461b7899" + revision_from: "v148, the tarball the Flathub manifest dev.ares.ares builds at 1259ea09 (EmuDeck installs the Flathub app)" + source_ref: "desktop-ui/desktop-ui.cpp:9-35, 100-103, 114, desktop-ui/settings/settings.cpp:31-35, 42, 223-235, nall/nall/path.cpp:123-142" + keys: + - key: "ColecoVision/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/colecovision.cpp:10-12, 42-50, mia/system/colecovision.cpp:7-9" + - key: "MSX/Firmware/BIOS.Japan" + source_ref: "desktop-ui/emulator/msx.cpp:16-19, 44-53" + - key: "FamicomDiskSystem/Firmware/BIOS.Japan" + source_ref: "desktop-ui/emulator/famicom-disk-system.cpp:17-19, 89-97" + - key: + - "Nintendo64DD/Firmware/BIOS.Japan" + - "Nintendo64DD/Firmware/BIOS.US" + - "Nintendo64DD/Firmware/BIOS.DEV" + source_ref: "desktop-ui/emulator/nintendo-64dd.cpp:17-19, 59-79" + condition: "the game's region" + - key: "GameBoyAdvance/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/game-boy-advance.cpp:12-14, 55-63" + - key: + - "MasterSystem/Firmware/BIOS.US" + - "MasterSystem/Firmware/BIOS.Japan" + - "MasterSystem/Firmware/BIOS.Europe" + source_ref: "desktop-ui/emulator/master-system.cpp:12-16, 93-107" + condition: "the game's region" + - key: "GameGear/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/game-gear.cpp:10-12, 30-40" + - key: + - "MegaCD/Firmware/BIOS.US" + - "MegaCD/Firmware/BIOS.Japan" + - "MegaCD/Firmware/BIOS.Europe" + source_ref: "desktop-ui/emulator/mega-cd.cpp:17-19, 63-83" + condition: "the disc's region" + - key: "NeoGeoAES/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/neo-geo-aes.cpp:14, 37-51, mia/system/neo-geo-aes.cpp:8-34" + condition: "a .zip path is searched for neo-epo.bin, any other path is read whole" + - key: "NeoGeoPocket/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/neo-geo-pocket.cpp:10-12, 32-40" + - key: "NeoGeoPocketColor/Firmware/BIOS.World" + source_ref: "desktop-ui/emulator/neo-geo-pocket-color.cpp:10-12, 32-40" + - key: + - "PlayStation/Firmware/BIOS.US" + - "PlayStation/Firmware/BIOS.Japan" + - "PlayStation/Firmware/BIOS.Europe" + source_ref: "desktop-ui/emulator/playstation.cpp:18-20, 78-98" + condition: "the disc's region" notes: | Multi-system emulator by Near (byuu), successor to higan/bsnes. Focuses on diff --git a/emulators/gzdoom.yml b/emulators/gzdoom.yml index d958edfa8..92672ba40 100644 --- a/emulators/gzdoom.yml +++ b/emulators/gzdoom.yml @@ -27,6 +27,18 @@ config_reads: source_ref: "src/common/utility/findfile.cpp:224-260" - key: "[SoundfontSearch.Directories] Path" source_ref: "src/common/audio/music/i_soundfont.cpp:393-410" + - file: "gzdoom.ini" + platforms: [retrodeck] + revision: "99aa489d09015a95bb78df2b30ede29f328cc874" + revision_from: "GZDoom g4.14.2-m in the gzdoom binary RetroDECK's components carry at fcd0d729 (gzdoom/rd_assets/gzdoom_release/bin/gzdoom), the g4.14.2 tag" + source_ref: "src/gameconfigfile.cpp:89-102, 894-904, src/common/platform/posix/unix/i_specialpaths.cpp:166-169" + keys: + - key: "[IWADSearch.Directories] Path" + source_ref: "src/d_iwad.cpp:457-471" + - key: "[FileSearch.Directories] Path" + source_ref: "src/common/utility/findfile.cpp:224-260" + - key: "[SoundfontSearch.Directories] Path" + source_ref: "src/common/audio/music/i_soundfont.cpp:393-410" notes: | Doom-engine source port by Christoph Oelckers. Supports Doom, Heretic, diff --git a/emulators/lrps2.yml b/emulators/lrps2.yml index d8541e5d8..22c17833d 100644 --- a/emulators/lrps2.yml +++ b/emulators/lrps2.yml @@ -30,6 +30,15 @@ config_reads: keys: - key: "pcsx2_bios" source_ref: "libretro/main.cpp:525-536, 2552-2560" + - file: "core options" + platforms: [retrodeck] + revision: "14d19f8339e3646b645c1c33e612b1888e632145" + revision_from: "library_version 14d19f8 of pcsx2_libretro.so in the buildbot's stable 1.22.2 RetroArch_cores.7z (2025-11-20), which RetroDECK's retroarch component installs (retroarch/component_recipe.json at fcd0d729)" + source_ref: "libretro/main.cpp:341, 1915-1926, 2013" + keys: + - key: "pcsx2_bios" + source_ref: "libretro/main.cpp:360-365, 1801-1830, pcsx2/VMManager.cpp:247-256, 1076-1092, 601, pcsx2/Pcsx2Config.cpp:887-891, 995-1000, pcsx2/ps2/BiosTools.cpp:267-281" + condition: "the value is a file of system/pcsx2/bios; without the option the core scans that folder for a 4 to 8 MiB BIOS" notes: | Hard fork of PCSX2 (pre-Qt, 1.7-dev era) ported to libretro, x86_64 only, diff --git a/scripts/launchers.py b/scripts/launchers.py index 138014337..3111f4070 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -369,13 +369,17 @@ def pool_folders(profile: dict) -> dict[str, str]: def _reads_apply(read: dict, profile: dict, platform: str) -> bool: """Whether a config_reads entry describes the build ``platform`` ships. - An entry naming platforms is read at their pin_build revision. One that - names none is read at the profile's pin, the build of every platform - with no pin_build of its own or one pinning the same commit. + An entry naming platforms is read at their pin_build revision or its + revision. One that names none is read at the profile's pin, the build + of every platform no entry names and with no pin_build of its own or + one pinning the same commit. """ named = [str(p) for p in read.get("platforms") or []] if named: return platform in named + # A platform an entry names has its build read there, not at the pin. + if any(platform in (r.get("platforms") or []) for r in profile.get("config_reads") or []): + return False pin = str(profile.get("source_commit") or "") builds = [b for b in profile.get("pin_build") or [] if b.get("platform") == platform] return all(pin and str(b.get("value") or "").startswith(pin[:12]) for b in builds) diff --git a/tests/test_launcher_evidence.py b/tests/test_launcher_evidence.py index bce6fe7e7..18dbe8074 100644 --- a/tests/test_launcher_evidence.py +++ b/tests/test_launcher_evidence.py @@ -265,6 +265,23 @@ class Invariants(unittest.TestCase): self.assertEqual(_config_reads_without_build(path, {"config_reads": closed}), []) +class ShippedBuild(unittest.TestCase): + def test_a_platform_an_entry_names_is_read_there_only(self) -> None: + """RetroDECK ships GZDoom 4.14.2: the entry read at the pin does not + also stand for it, so a key only the pin reads proves nothing there.""" + duck = profile(config_reads=[ + {"file": "settings.ini", "source_ref": "src/core.cpp:10", + "keys": [{"key": "[BIOS] SearchDirectory", "source_ref": "src/settings.cpp:20"}, + {"key": ["[BIOS] PathPAL", "[BIOS] PathNTSCU"], "source_ref": "src/bios.cpp:30"}]}, + {"file": "settings.ini", "platforms": ["rocknix"], "revision": "a" * 40, + "revision_from": "the AppImage", "source_ref": "src/core.cpp:12", + "keys": [{"key": "[BIOS] SearchDirectory", "source_ref": "src/settings.cpp:22"}]}, + ]) + evidence = config_evidence(record(), {"duck": duck}, "rocknix") + self.assertEqual(evidence.missing, ("config_reads: no read of [BIOS] PathPAL",)) + self.assertTrue(config_evidence(record(), {"duck": duck}, "emuelec").proven) + + class CoreOptions(unittest.TestCase): def test_a_core_option_is_read_by_the_core_not_the_frontend(self) -> None: """RetroDECK's retroarch component writes pcsx2_bios: the frontend