feat: make the native export round-trip a platform file

scripts/scraper/base_scraper.py

@@ -28,6 +28,15 @@ class BiosRequirement:
     required: bool = True
     zipped_file: str | None = None  # If set, md5 is for this ROM inside the ZIP
     native_id: str | None = None  # Original system name before normalization
+    sha256: str | None = None
+    alt_md5: str | None = None
+    # How the platform itself writes the file reference. Its own spelling
+    # is not always derivable from ours: libretro writes iplromco.dat bare
+    # but ep128emu/roms/cpc464.rom with its directory.
+    native_path: str | None = None
+    # Fields the platform declares that have no equivalent in our model.
+    # Kept verbatim so the native file can be written back unchanged.
+    native: dict[str, object] = field(default_factory=dict)
 
 
 @dataclass
@@ -58,6 +67,40 @@ class ChangeSet:
 MAX_RESPONSE_SIZE = 50 * 1024 * 1024  # 50 MB
 
 
+def requirement_entry(req: BiosRequirement) -> dict:
+    """Serialize a requirement to a platform YAML file entry.
+
+    Carries the platform's own system id and any field that has no place in
+    our model. Without them the transcription is lossy in one direction:
+    several native systems collapse onto one slug, and the exporter can no
+    longer tell which of them a file came from.
+    """
+    entry: dict = {
+        "name": req.name,
+        "destination": req.destination,
+        "required": req.required,
+    }
+    for field_name in ("sha1", "md5", "sha256", "crc32"):
+        value = getattr(req, field_name, None)
+        if value:
+            entry[field_name] = str(value).lower()
+    if req.size:
+        entry["size"] = req.size
+    if req.zipped_file:
+        entry["zipped_file"] = req.zipped_file
+    if req.alt_md5:
+        entry["alt_md5"] = str(req.alt_md5).lower()
+    if req.native_id:
+        entry["native_system"] = req.native_id
+    if req.native_path and req.native_path != req.name:
+        entry["native_path"] = req.native_path
+    for key in sorted(req.native):
+        value = req.native[key]
+        if value not in (None, "", [], {}):
+            entry[key] = value
+    return entry
+
+
 def _read_limited(resp: object, max_bytes: int = MAX_RESPONSE_SIZE) -> bytes:
     """Read an HTTP response with a size limit to prevent OOM."""
     chunks: list[bytes] = []

scripts/scraper/batocera_scraper.py

@@ -19,7 +19,7 @@ from pathlib import Path
 
 from common import yaml_load
 
-from .base_scraper import BaseScraper, BiosRequirement
+from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
 
 PLATFORM_NAME = "batocera"
 
@@ -309,7 +309,8 @@ class Scraper(BaseScraper):
             bios_files = sys_data.get("biosFiles", [])
 
             for bios in bios_files:
-                file_path = bios.get("file", "")
+                declared_path = bios.get("file", "")
+                file_path = declared_path
                 md5 = _resolve_truncated_md5(bios.get("md5", ""), md5_index)
                 zipped_file = bios.get("zippedFile", "")
 
@@ -324,9 +325,11 @@ class Scraper(BaseScraper):
                         system=system_slug,
                         md5=md5 or None,
                         destination=file_path,
+                        native_path=declared_path,
                         required=True,
                         zipped_file=zipped_file or None,
                         native_id=sys_key,
+                        native={"native_name": sys_data.get("name", "")},
                     )
                 )
 
@@ -365,17 +368,7 @@ class Scraper(BaseScraper):
                     sys_entry["name"] = dname
                 systems[req.system] = sys_entry
 
-            entry = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                entry["md5"] = req.md5
-            if req.zipped_file:
-                entry["zipped_file"] = req.zipped_file
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         batocera_version = ""
         if _STABLE_TAG != "master":

scripts/scraper/bizhawk_scraper.py

@@ -22,6 +22,7 @@ import re
 
 try:
     from .base_scraper import (
+        requirement_entry,
         BaseScraper,
         BiosRequirement,
         fetch_github_latest_version,
@@ -29,6 +30,7 @@ try:
     )
 except ImportError:
     from base_scraper import (
+        requirement_entry,
         BaseScraper,
         BiosRequirement,
         fetch_github_latest_version,
@@ -352,6 +354,8 @@ class Scraper(BaseScraper):
                 sha1=rec["sha1"],
                 size=rec["size"] if rec["size"] else None,
                 required=rec.get("status") != "Bad",
+                destination=rec["name"],
+                native_id=rec["system"],
             )
             requirements.append(req)
 
@@ -366,17 +370,7 @@ class Scraper(BaseScraper):
             if req.system not in systems:
                 systems[req.system] = {"files": []}
 
-            entry: dict = {
-                "name": req.name,
-                "destination": req.name,
-                "required": req.required,
-            }
-            if req.sha1:
-                entry["sha1"] = req.sha1.lower()
-            if req.size:
-                entry["size"] = req.size
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         version = _STABLE_TAG if _STABLE_TAG != "master" else ""
 

scripts/scraper/emudeck_scraper.py

@@ -18,9 +18,19 @@ import urllib.error
 import urllib.request
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "emudeck"
 
@@ -54,9 +64,18 @@ HASH_ARRAY_MAP = {
     "SaturnBios": "sega-saturn",
 }
 
+# Every check checkBIOS.sh defines, and the system it stands for. A function
+# missing here is a check whose hashes nothing would carry.
 FUNCTION_HASH_MAP = {
+    "checkPS1BIOS": "sony-playstation",
+    "checkPS2BIOS": "sony-playstation-2",
+    "checkSegaCDBios": "sega-mega-cd",
+    "checkSaturnBios": "sega-saturn",
     "checkDreamcastBios": "sega-dreamcast",
     "checkDSBios": "nintendo-ds",
+    "checkCitronBios": "nintendo-switch",
+    "checkRyujinxBios": "nintendo-switch",
+    "checkYuzuBios": "nintendo-switch",
 }
 
 SYSTEM_SLUG_MAP = {
@@ -171,13 +190,17 @@ _RE_ARRAY = re.compile(
     re.MULTILINE,
 )
 
+# checkBIOS.sh declares its checks as `checkPS1BIOS(){`, with no `function`
+# keyword and no settled casing for BIOS.
 _RE_FUNC = re.compile(
-    r"function\s+(check\w+Bios)\s*\(\)",
+    r"^[ \t]*(?:function\s+)?(check\w*(?:BIOS|Bios))\s*\(\)\s*\{",
     re.MULTILINE,
 )
 
+# The hash list is named differently in each check (PSBios, hashes, ...), so
+# it is found by shape rather than by name.
 _RE_LOCAL_HASHES = re.compile(
-    r"local\s+hashes=\(\s*((?:[0-9a-fA-F]+\s*)+)\)",
+    r"(?:local\s+)?\w+=\(\s*((?:[0-9a-fA-F]{32}\s*)+)\)",
     re.MULTILINE,
 )
 
@@ -346,6 +369,7 @@ class Scraper(BaseScraper):
                         system=system,
                         destination=f.get("destination", f["name"]),
                         required=True,
+                        native_id=system,
                     )
                 )
 
@@ -357,6 +381,7 @@ class Scraper(BaseScraper):
                         md5=md5,
                         destination="",
                         required=True,
+                        native_id=system,
                     )
                 )
 
@@ -379,6 +404,7 @@ class Scraper(BaseScraper):
                             system=system,
                             destination=f.get("destination", f["name"]),
                             required=True,
+                            native_id=system,
                         )
                     )
 
@@ -398,14 +424,7 @@ class Scraper(BaseScraper):
             if req.system not in systems:
                 systems[req.system] = {"files": []}
 
-            entry: dict = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                entry["md5"] = req.md5
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         version = ""
         try:

scripts/scraper/libretro_scraper.py

@@ -11,7 +11,12 @@ from __future__ import annotations
 import urllib.error
 import urllib.request
 
-from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+from .base_scraper import (
+    BaseScraper,
+    BiosRequirement,
+    fetch_github_latest_version,
+    requirement_entry,
+)
 from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
 
 PLATFORM_NAME = "libretro"
@@ -132,6 +137,7 @@ class Scraper(BaseScraper):
                     destination=destination,
                     required=True,
                     native_id=native_system,
+                    native_path=rom.name,
                 )
             )
 
@@ -247,21 +253,7 @@ class Scraper(BaseScraper):
                         system_entry["docs"] = cm["docs"]
                 systems[req.system] = system_entry
 
-            entry = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.sha1:
-                entry["sha1"] = req.sha1
-            if req.md5:
-                entry["md5"] = req.md5
-            if req.crc32:
-                entry["crc32"] = req.crc32
-            if req.size:
-                entry["size"] = req.size
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         # Systems not in System.dat but needed for RetroArch -added via
         # shared groups in _shared.yml. The includes directive is resolved

scripts/scraper/misterfpga_scraper.py

@@ -29,9 +29,19 @@ import zipfile
 from datetime import datetime, timezone
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, _read_limited
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        _read_limited,
+        requirement_entry,
+    )
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement, _read_limited
+    from base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        _read_limited,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "misterfpga"
 
@@ -193,15 +203,7 @@ class Scraper(BaseScraper):
                     "docs": f"https://github.com/MiSTer-devel/{repo}" if repo else "",
                 },
             )
-            file_entry: dict = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-                "md5": req.md5,
-            }
-            if req.size is not None:
-                file_entry["size"] = req.size
-            entry["files"].append(file_entry)
+            entry["files"].append(requirement_entry(req))
 
         for entry in systems.values():
             if not entry["docs"]:

scripts/scraper/recalbox_scraper.py

@@ -15,11 +15,9 @@ Recalbox verification logic:
 
 from __future__ import annotations
 
-import sys
-
 from common import parse_untrusted_xml
 
-from .base_scraper import BaseScraper, BiosRequirement
+from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
 
 PLATFORM_NAME = "recalbox"
 
@@ -136,6 +134,7 @@ class Scraper(BaseScraper):
         for system_elem in root.findall(".//system"):
             platform = system_elem.get("platform", "")
             system_slug = SYSTEM_SLUG_MAP.get(platform, platform)
+            fullname = system_elem.get("fullname", "")
 
             for bios_elem in system_elem.findall("bios"):
                 paths_str = bios_elem.get("path", "")
@@ -154,11 +153,30 @@ class Scraper(BaseScraper):
                 md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
                 all_md5 = ",".join(md5_list) if md5_list else None
 
-                dedup_key = primary_path
+                dedup_key = (platform, primary_path)
                 if dedup_key in seen:
                     continue
                 seen.add(dedup_key)
 
+                native: dict[str, object] = {}
+                if fullname:
+                    native["native_name"] = fullname
+                core = bios_elem.get("core", "").strip()
+                if core:
+                    native["core"] = core
+                note = bios_elem.get("note", "").strip()
+                if note:
+                    native["note"] = note
+                # Recalbox reads a missing attribute as true for both flags,
+                # so only the explicit value carries information.
+                hash_match = bios_elem.get("hashMatchMandatory")
+                if hash_match is not None:
+                    native["hash_match_mandatory"] = hash_match != "false"
+                if bios_elem.get("mandatory") is not None:
+                    native["mandatory_declared"] = mandatory
+                if len(paths) > 1:
+                    native["alt_paths"] = paths[1:]
+
                 requirements.append(
                     BiosRequirement(
                         name=name,
@@ -167,56 +185,12 @@ class Scraper(BaseScraper):
                         destination=primary_path,
                         required=mandatory,
                         native_id=platform,
+                        native=native,
                     )
                 )
 
         return requirements
 
-    def fetch_full_requirements(self) -> list[dict]:
-        """Parse es_bios.xml preserving all Recalbox-specific fields."""
-        raw = self._fetch_raw()
-        root = parse_untrusted_xml(raw, "es_bios.xml")
-        requirements = []
-
-        for system_elem in root.findall(".//system"):
-            platform = system_elem.get("platform", "")
-            system_name = system_elem.get("name", platform)
-            system_slug = SYSTEM_SLUG_MAP.get(platform, platform)
-
-            for bios_elem in system_elem.findall("bios"):
-                paths_str = bios_elem.get("path", "")
-                md5_str = bios_elem.get("md5", "")
-                core = bios_elem.get("core", "")
-                mandatory = bios_elem.get("mandatory", "true") != "false"
-                hash_match_mandatory = (
-                    bios_elem.get("hashMatchMandatory", "true") != "false"
-                )
-                note = bios_elem.get("note", "")
-
-                paths = [p.strip() for p in paths_str.split("|") if p.strip()]
-                md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
-
-                if not paths:
-                    continue
-
-                name = paths[0].split("/")[-1] if "/" in paths[0] else paths[0]
-
-                requirements.append(
-                    {
-                        "name": name,
-                        "system": system_slug,
-                        "system_name": system_name,
-                        "paths": paths,
-                        "md5_list": md5_list,
-                        "core": core,
-                        "mandatory": mandatory,
-                        "hash_match_mandatory": hash_match_mandatory,
-                        "note": note,
-                    }
-                )
-
-        return requirements
-
     def validate_format(self, raw_data: str) -> bool:
         """Validate es_bios.xml format."""
         return "<biosList" in raw_data and "<system" in raw_data and "<bios" in raw_data
@@ -225,7 +199,7 @@ class Scraper(BaseScraper):
         """Generate a platform YAML config dict from scraped data."""
         requirements = self.fetch_requirements()
 
-        systems = {}
+        systems: dict[str, dict] = {}
         for req in requirements:
             if req.system not in systems:
                 sys_entry: dict = {"files": []}
@@ -233,15 +207,7 @@ class Scraper(BaseScraper):
                     sys_entry["native_id"] = req.native_id
                 systems[req.system] = sys_entry
 
-            entry = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                entry["md5"] = req.md5
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         version = _STABLE_TAG if _STABLE_TAG != "master" else ""
         if not version:
@@ -262,55 +228,9 @@ class Scraper(BaseScraper):
 
 def main():
     """CLI entry point."""
-    import argparse
-    import json
+    from .base_scraper import scraper_cli
 
-    parser = argparse.ArgumentParser(description="Scrape Recalbox es_bios.xml")
-    parser.add_argument("--dry-run", action="store_true")
-    parser.add_argument("--json", action="store_true")
-    parser.add_argument(
-        "--full", action="store_true", help="Show full Recalbox-specific fields"
-    )
-    parser.add_argument("--output", "-o")
-    args = parser.parse_args()
-
-    scraper = Scraper()
-
-    try:
-        if args.full:
-            reqs = scraper.fetch_full_requirements()
-            print(json.dumps(reqs[:5], indent=2))
-            print(f"\nTotal: {len(reqs)} BIOS entries")
-            return
-        reqs = scraper.fetch_requirements()
-    except (ConnectionError, ValueError) as e:
-        print(f"Error: {e}", file=sys.stderr)
-        sys.exit(1)
-
-    if args.dry_run:
-        from collections import defaultdict
-
-        by_system = defaultdict(list)
-        for r in reqs:
-            by_system[r.system].append(r)
-        for sys_name, files in sorted(by_system.items()):
-            print(f"\n{sys_name} ({len(files)} files):")
-            for f in files[:5]:
-                print(f"  {f.name} (md5={f.md5[:12] if f.md5 else 'N/A'}...)")
-            if len(files) > 5:
-                print(f"  ... +{len(files) - 5} more")
-        print(f"\nTotal: {len(reqs)} BIOS files across {len(by_system)} systems")
-        return
-
-    if args.json:
-        config = scraper.generate_platform_yaml()
-        print(json.dumps(config, indent=2))
-        return
-
-    by_system = {}
-    for r in reqs:
-        by_system.setdefault(r.system, []).append(r)
-    print(f"Scraped {len(reqs)} BIOS files across {len(by_system)} systems")
+    scraper_cli(Scraper, "Scrape Recalbox es_bios.xml")
 
 
 if __name__ == "__main__":

scripts/scraper/retrobat_scraper.py

@@ -11,9 +11,19 @@ from __future__ import annotations
 import json
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "retrobat"
 
@@ -73,7 +83,8 @@ class Scraper(BaseScraper):
                 if not isinstance(bios, dict):
                     continue
 
-                file_path = bios.get("file", "")
+                declared_path = bios.get("file", "")
+                file_path = declared_path
                 md5 = bios.get("md5", "")
 
                 if not file_path:
@@ -85,6 +96,11 @@ class Scraper(BaseScraper):
 
                 name = file_path.split("/")[-1] if "/" in file_path else file_path
 
+                native: dict[str, object] = {}
+                sys_name = sys_data.get("name", "") if isinstance(sys_data, dict) else ""
+                if sys_name:
+                    native["native_name"] = sys_name
+
                 requirements.append(
                     BiosRequirement(
                         name=name,
@@ -92,6 +108,9 @@ class Scraper(BaseScraper):
                         md5=md5 or None,
                         destination=file_path,
                         required=True,
+                        native_id=sys_key,
+                        native_path=declared_path,
+                        native=native,
                     )
                 )
 
@@ -139,15 +158,7 @@ class Scraper(BaseScraper):
                     sys_entry["name"] = dname
                 systems[req.system] = sys_entry
 
-            entry = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                entry["md5"] = req.md5
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         version = ""
         tag = fetch_github_latest_version(GITHUB_REPO)

scripts/scraper/retrodeck_scraper.py

@@ -36,10 +36,10 @@ import urllib.request
 from pathlib import Path
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement
+    from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
 except ImportError:
     sys.path.insert(0, str(Path(__file__).parent.parent))
-    from scraper.base_scraper import BaseScraper, BiosRequirement
+    from scraper.base_scraper import BaseScraper, BiosRequirement, requirement_entry
 
 PLATFORM_NAME = "retrodeck"
 COMPONENTS_REPO = "RetroDECK/components"
@@ -385,13 +385,25 @@ class Scraper(BaseScraper):
                         continue
                     seen.add(key)
 
+                    native: dict[str, object] = {"component": comp_key}
+                    description = str(entry.get("description", "")).strip()
+                    if description:
+                        native["description"] = description
+                    if required_raw not in (None, ""):
+                        native["required_label"] = str(required_raw)
+
+                    sha256 = str(entry.get("sha256", "")).strip().lower()
+
                     requirements.append(
                         BiosRequirement(
                             name=filename,
                             system=system,
                             destination=destination,
                             md5=md5,
+                            sha256=sha256 or None,
                             required=required,
+                            native_id=str(raw_system),
+                            native=native,
                         )
                     )
 
@@ -413,14 +425,7 @@ class Scraper(BaseScraper):
         systems: dict[str, dict] = {}
         for req in reqs:
             sys_entry = systems.setdefault(req.system, {"files": []})
-            file_entry: dict = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                file_entry["md5"] = req.md5
-            sys_entry["files"].append(file_entry)
+            sys_entry["files"].append(requirement_entry(req))
 
         try:
             from .base_scraper import fetch_github_latest_version

scripts/scraper/retropie_scraper.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Scraper for RetroPie's package list.
+
+Source: RetroPie/RetroPie-Setup -> scriptmodules/
+Format: one bash script per package, `rp_module_id` naming it
+
+RetroPie publishes no BIOS list: platforms.cfg carries extensions and full
+names, and the files a package needs are named in prose inside its
+`rp_module_help`, without hashes. So there is nothing here to transcribe as
+requirements, and `fetch_requirements` returns none. What RetroPie does
+state precisely is which packages it ships, and that is what this reads.
+
+It matters because the config said `cores: all_libretro`, inherited from
+RetroArch: RetroPie claimed every libretro core in existence while shipping
+96 of them, and claimed none of the standalone emulators it also packages
+(openmsx, pcsx2, xroar, sdltrs, amiberry and the rest). Both halves were
+wrong, in opposite directions.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+import tarfile
+import urllib.error
+import urllib.request
+
+try:
+    from .base_scraper import BaseScraper, BiosRequirement
+except ImportError:
+    from base_scraper import BaseScraper, BiosRequirement
+
+PLATFORM_NAME = "retropie"
+
+SOURCE_URL = (
+    "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
+)
+GITHUB_REPO = "RetroPie/RetroPie-Setup"
+MAX_ARCHIVE = 64 * 1024 * 1024
+
+_MODULE_ID = re.compile(r'^rp_module_id="([^"]+)"', re.MULTILINE)
+# Sections RetroPie does not build as emulators: setup helpers, themes,
+# drivers and the like carry no core.
+_PACKAGE_DIRS = ("emulators", "libretrocores", "ports")
+
+
+class Scraper(BaseScraper):
+    """Scraper for the RetroPie package list."""
+
+    def __init__(self, url: str = SOURCE_URL):
+        super().__init__(url=url)
+        self._modules: list[str] | None = None
+
+    def _fetch_archive(self) -> bytes:
+        request = urllib.request.Request(
+            self.url, headers={"User-Agent": "retrobios-scraper/1.0"}
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=60) as response:
+                payload = response.read(MAX_ARCHIVE + 1)
+        except urllib.error.URLError as exc:
+            raise ConnectionError(f"Failed to fetch {self.url}: {exc}") from exc
+        if len(payload) > MAX_ARCHIVE:
+            raise ValueError(f"{self.url}: response larger than {MAX_ARCHIVE} bytes")
+        return payload
+
+    def module_ids(self, payload: bytes | None = None) -> list[str]:
+        """Every package RetroPie ships, by the id its script declares."""
+        if self._modules is not None:
+            return self._modules
+
+        raw = payload if payload is not None else self._fetch_archive()
+        found: set[str] = set()
+        with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as archive:
+            for member in archive:
+                if not member.isfile() or not member.name.endswith(".sh"):
+                    continue
+                relative = member.name.split("/", 1)[-1]
+                parts = relative.split("/")
+                if len(parts) != 3 or parts[0] != "scriptmodules":
+                    continue
+                if parts[1] not in _PACKAGE_DIRS:
+                    continue
+                handle = archive.extractfile(member)
+                if handle is None:
+                    continue
+                text = handle.read().decode("utf-8", errors="replace")
+                match = _MODULE_ID.search(text)
+                if match:
+                    found.add(match.group(1))
+
+        self._modules = sorted(found)
+        return self._modules
+
+    def fetch_requirements(self) -> list[BiosRequirement]:
+        """None: RetroPie names BIOS in prose, with no hash to transcribe.
+
+        The prose is read where it can be acted on, by the exporter that
+        rewrites those sentences.
+        """
+        self.module_ids()
+        return []
+
+    def validate_format(self, raw_data: str) -> bool:
+        return bool(self.module_ids())
+
+    def generate_platform_yaml(self) -> dict:
+        """Build the RetroPie platform configuration.
+
+        The systems stay inherited from RetroArch: RetroPie installs the
+        libretro cores and reads the same files, at BIOS/ instead of
+        system/. Only the core list is its own.
+        """
+        # A libretro package is lr-<core>; a standalone package is named
+        # after the emulator itself.
+        cores = sorted({module.removeprefix("lr-") for module in self.module_ids()})
+
+        return {
+            "inherits": "retroarch",
+            "platform": "RetroPie",
+            "homepage": "https://retropie.org.uk",
+            "source": SOURCE_URL,
+            "base_destination": "BIOS",
+            "cores": cores,
+        }
+
+
+def main() -> None:
+    try:
+        from .base_scraper import scraper_cli
+    except ImportError:
+        from base_scraper import scraper_cli
+
+    scraper_cli(Scraper, "Scrape the RetroPie package list")
+
+
+if __name__ == "__main__":
+    main()

scripts/scraper/rocknix_scraper.py

@@ -18,9 +18,19 @@ from __future__ import annotations
 import re
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "rocknix"
 
@@ -136,16 +146,15 @@ class Scraper(BaseScraper):
                 # Declared paths are relative to the storage root: strip the
                 # leading bios/ so destinations match base_destination
                 destination = path[len("bios/"):] if path.startswith("bios/") else path
-                md5_values = [bios.get("md5", ""), bios.get("altmd5", "")]
-                md5 = ",".join(m for m in md5_values if m) or None
-
                 req = BiosRequirement(
                     name=destination.rsplit("/", 1)[-1],
                     system=system_slug,
-                    md5=md5,
+                    md5=bios.get("md5", "") or None,
+                    alt_md5=bios.get("altmd5", "") or None,
                     destination=destination,
                     required=True,
                     native_id=key,
+                    native={"native_name": entry["name"]},
                 )
                 if bios.get("zippedFile"):
                     req.zipped_file = bios["zippedFile"]
@@ -168,16 +177,7 @@ class Scraper(BaseScraper):
                     "name": names.get(req.native_id, req.system),
                 },
             )
-            file_entry: dict = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.md5:
-                file_entry["md5"] = req.md5
-            if getattr(req, "zipped_file", None):
-                file_entry["zipped_file"] = req.zipped_file
-            entry["files"].append(file_entry)
+            entry["files"].append(requirement_entry(req))
 
         return {
             "platform": "ROCKNIX",

scripts/scraper/romm_scraper.py

@@ -26,9 +26,19 @@ import json
 import sys
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "romm"
 
@@ -165,6 +175,7 @@ class Scraper(BaseScraper):
                         size=size,
                         destination=f"{slug}/{filename}",
                         required=True,
+                        native_id=slug,
                     )
                 )
 
@@ -183,7 +194,6 @@ class Scraper(BaseScraper):
         for key in list(data.keys())[:5]:
             if ":" not in key:
                 return False
-            _, _entry = key.split(":", 1), data[key]
             if not isinstance(data[key], dict):
                 return False
             if "md5" not in data[key] and "sha1" not in data[key]:
@@ -200,21 +210,7 @@ class Scraper(BaseScraper):
             if req.system not in systems:
                 systems[req.system] = {"files": []}
 
-            entry: dict = {
-                "name": req.name,
-                "destination": req.destination,
-                "required": req.required,
-            }
-            if req.sha1:
-                entry["sha1"] = req.sha1
-            if req.md5:
-                entry["md5"] = req.md5
-            if req.crc32:
-                entry["crc32"] = req.crc32
-            if req.size:
-                entry["size"] = req.size
-
-            systems[req.system]["files"].append(entry)
+            systems[req.system]["files"].append(requirement_entry(req))
 
         version = _STABLE_TAG if _STABLE_TAG != "master" else ""