feat: record the systems each libretro core runs

scripts/scraper/batocera_scraper.py

@@ -142,6 +142,29 @@ def es_systems_libretro(text: str) -> set[str]:
     return found
 
 
+def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[str]]:
+    """libretro core -> the systems es_systems.yml offers it under `libretro:`.
+
+    configgen hands a libretro core /userdata/saves/<system> at each launch
+    (libretroConfig.py: savefile_directory = SAVES / system.name), so a file
+    a core reads below its save directory sits once in each of these
+    system folders.
+    """
+    data = yaml_load_upstream(text)
+    if not isinstance(data, dict):
+        raise ValueError("es_systems.yml is not a mapping")
+    found: dict[str, set[str]] = {}
+    for system, spec in data.items():
+        emulators = spec.get("emulators") if isinstance(spec, dict) else None
+        libretro = emulators.get("libretro") if isinstance(emulators, dict) else None
+        if not isinstance(libretro, dict):
+            continue
+        for core, entry in libretro.items():
+            if isinstance(entry, dict) and str(core) not in dropped:
+                found.setdefault(str(core), set()).add(str(system))
+    return {core: sorted(systems) for core, systems in sorted(found.items())}
+
+
 def es_systems_by_system(
     es_systems_text: str, defaults_text: str, dropped: set[str]
 ) -> dict[str, list[str]]:
@@ -518,6 +541,9 @@ class Scraper(BaseScraper):
         )
 
         cores, standalone, dual = self._fetch_cores()
+        libretro_systems = es_systems_libretro_systems(
+            self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
+        )
         result = {
             "platform": "Batocera",
             "version": self.tag.removeprefix("batocera-"),
@@ -534,6 +560,8 @@ class Scraper(BaseScraper):
         if dual:
             # Launched both ways: each layout gets its files.
             result["dual_mode_cores"] = dual
+        # The systems each libretro core runs for: its save folders.
+        result["libretro_systems"] = libretro_systems
         result["launchers"] = extract_launchers(
             lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
         )

scripts/scraper/knulli_scraper.py

@@ -38,6 +38,7 @@ from .batocera_scraper import (
     _not_cores,
     configgen_cores,
     es_systems_by_system,
+    es_systems_libretro_systems,
     extract_systems_dict,
     platform_systems,
     requirements_from_systems,
@@ -120,6 +121,9 @@ class Scraper(BaseScraper):
             result["standalone_cores"] = standalone
         if dual:
             result["dual_mode_cores"] = dual
+        # configgen hands a libretro core /userdata/saves/<system>, as
+        # Batocera's does: the systems each core runs for are its save folders.
+        result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
         return result
 
 

scripts/scraper/recalbox_scraper.py

@@ -167,6 +167,29 @@ def system_ini_libretro(text: str) -> set[str]:
     }
 
 
+def system_ini_name(text: str, folder: str) -> str:
+    """The system name a system.ini gives, the one ES passes configgen.
+
+    configgen hands a libretro core /recalbox/share/saves/<that name>
+    (projects/configgen/configgen/generators/libretro/libretroRetroarch.py
+    savesFolder). The folder name stands in where the file names none.
+    """
+    return _ini_sections(text).get("system", {}).get("name") or folder
+
+
+def libretro_systems(inis: dict[str, str]) -> dict[str, list[str]]:
+    """libretro core -> the systems whose system.ini offers it.
+
+    ``inis`` maps each system folder to its system.ini.
+    """
+    found: dict[str, set[str]] = {}
+    for folder, text in inis.items():
+        name = system_ini_name(text, folder)
+        for core in system_ini_libretro(text):
+            found.setdefault(core, set()).add(name)
+    return {core: sorted(systems) for core, systems in sorted(found.items())}
+
+
 def system_ini_cores(text: str) -> tuple[set[str], set[str]]:
     """The cores one Recalbox system.ini offers, and those run standalone.
 
@@ -282,7 +305,10 @@ class Scraper(BaseScraper):
         cores: set[str] = set()
         standalone: set[str] = set()
         libretro: set[str] = set()
-        for text in self._system_inis(systems):
+        texts = self._system_inis(systems)
+        # The systems each libretro core runs for: its save folders.
+        self.libretro_systems = libretro_systems(dict(zip(systems, texts)))
+        for text in texts:
             found, alone = system_ini_cores(text)
             cores |= found
             standalone |= alone
@@ -397,6 +423,7 @@ class Scraper(BaseScraper):
             # Launched both ways: each layout gets its files.
             **({"dual_mode_cores": dual} if dual else {}),
             "systems": systems,
+            "libretro_systems": self.libretro_systems,
             "launchers": self._fetch_launchers(),
         }
 

scripts/scraper/retrobat_scraper.py

@@ -99,6 +99,31 @@ def es_systems_cores(text: str) -> tuple[set[str], set[str]]:
     return cores, standalone
 
 
+def es_systems_libretro_systems(text: str) -> dict[str, list[str]]:
+    """libretro core -> the ES systems es_systems.cfg offers it for.
+
+    emulatorLauncher hands a libretro core saves\\<system> at each launch
+    (LibRetro.Generator.cs Configure: savefile_directory), so a file a core
+    reads below its save directory sits once in each of these system
+    folders. A commented-out system is no element and counts for nothing.
+    """
+    from common import parse_untrusted_xml
+
+    found: dict[str, set[str]] = {}
+    for system in parse_untrusted_xml(text, "es_systems.cfg").iter("system"):
+        name = (system.findtext("name") or "").strip()
+        if not name:
+            continue
+        for emulator in system.iter("emulator"):
+            if (emulator.get("name") or "").strip() != "libretro":
+                continue
+            for core in emulator.iter("core"):
+                core_name = (core.text or "").strip()
+                if core_name:
+                    found.setdefault(core_name, set()).add(name)
+    return {core: sorted(systems) for core, systems in sorted(found.items())}
+
+
 def launcher_revision(url: str) -> str:
     """The emulatorLauncher commit a source URL is pinned to."""
     found = re.search(r"/emulatorlauncher/([0-9a-f]{40})/", url)
@@ -259,6 +284,7 @@ class Scraper(BaseScraper):
                 es_text = resp.read().decode("utf-8")
             cores, standalone = es_systems_cores(es_text)
             dual = es_systems_dual(es_text)
+            libretro_systems = es_systems_libretro_systems(es_text)
         except urllib.error.URLError as e:
             raise ConnectionError(f"Failed to fetch {es_url}: {e}") from e
         if not cores:
@@ -277,6 +303,8 @@ class Scraper(BaseScraper):
             # Launched both ways: each layout gets its files.
             **({"dual_mode_cores": sorted(dual)} if dual else {}),
             "systems": systems,
+            # The systems each libretro core runs for: its save folders.
+            "libretro_systems": libretro_systems,
             "launchers": fetch_launchers(launcher_revision(self.url)),
         }
 

scripts/scraper/retrodeck_scraper.py

@@ -39,6 +39,8 @@ try:
         BaseScraper,
         BiosRequirement,
         fetch_branch_head,
+        fetch_bytes,
+        fetch_github_latest_version,
         github_headers,
         requirement_entry,
     )
@@ -49,6 +51,8 @@ except ImportError:
         BaseScraper,
         BiosRequirement,
         fetch_branch_head,
+        fetch_bytes,
+        fetch_github_latest_version,
         github_headers,
         requirement_entry,
     )
@@ -60,6 +64,38 @@ COMPONENTS_BRANCH = "main"
 COMPONENTS_API_URL = "https://api.github.com/repos/" + COMPONENTS_REPO + "/git/trees/{rev}"
 RAW_BASE = "https://raw.githubusercontent.com/" + COMPONENTS_REPO + "/{rev}"
 SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
+
+# ES-DE's system list as RetroDECK's fork builds it.
+ES_DE_REPO = "RetroDECK/ES-DE"
+ES_DE_SYSTEMS_URL = (
+    "https://raw.githubusercontent.com/" + ES_DE_REPO
+    + "/{rev}/resources/systems/linux/es_systems.xml"
+)
+_RETROARCH_CORE = re.compile(r"%CORE_RETROARCH%/([\w.+-]+?)_libretro\.so")
+
+
+def es_systems_libretro_systems(text: str) -> dict[str, list[str]]:
+    """libretro core -> the ROM folders of the systems whose commands load it.
+
+    RetroDECK's RetroArch sorts saves by content and not by core
+    (retroarch/rd_assets/rd_config/retroarch.cfg, sort_savefiles_by_content_enable
+    and sort_savefiles_enable): a core is handed saves/<folder holding the
+    ROM>, the system's <path> folder below %ROMPATH%. A commented-out
+    command is no element and counts for nothing.
+    """
+    from common import parse_untrusted_xml
+
+    found: dict[str, set[str]] = {}
+    for system in parse_untrusted_xml(text, "es_systems.xml").iter("system"):
+        path = (system.findtext("path") or "").strip().replace("\\", "/")
+        folder = path.rstrip("/").rsplit("/", 1)[-1]
+        if not folder or folder.startswith("%"):
+            continue
+        for command in system.iter("command"):
+            match = _RETROARCH_CORE.search(command.text or "")
+            if match:
+                found.setdefault(match.group(1), set()).add(folder)
+    return {core: sorted(folders) for core, folders in sorted(found.items())}
 NON_EMULATOR_COMPONENTS = {
     "framework",
     "es-de",
@@ -468,6 +504,16 @@ class Scraper(BaseScraper):
 
         return requirements
 
+    def _fetch_libretro_systems(self) -> dict[str, list[str]]:
+        """libretro core -> ROM folders, from the ES-DE build RetroDECK ships.
+
+        RetroDECK takes ES-DE's latest release of its fork
+        (automation-tools/alchemist/desired_versions.sh, ES_DE_DESIRED_VERSION).
+        """
+        tag = fetch_github_latest_version(ES_DE_REPO)
+        text = fetch_bytes(ES_DE_SYSTEMS_URL.format(rev=tag)).decode("utf-8")
+        return es_systems_libretro_systems(text)
+
     def generate_platform_yaml(self) -> dict:
         reqs = self.fetch_requirements()
         manifests = self._get_manifests()
@@ -486,10 +532,6 @@ class Scraper(BaseScraper):
             sys_entry = systems.setdefault(req.system, {"files": []})
             sys_entry["files"].append(requirement_entry(req))
 
-        try:
-            from .base_scraper import fetch_github_latest_version
-        except ImportError:
-            from scraper.base_scraper import fetch_github_latest_version
         version = fetch_github_latest_version("RetroDECK/RetroDECK")
 
         return {
@@ -509,6 +551,8 @@ class Scraper(BaseScraper):
             # its own, and retroarch is the one that runs cores.
             "standalone_cores": [c for c in cores if c != "retroarch"],
             "systems": systems,
+            # The ROM folders each libretro core runs for: its save folders.
+            "libretro_systems": self._fetch_libretro_systems(),
             "launchers": extract_launchers(cores, self.component_file),
         }