feat: scrape platform data from stable release tags

.github/workflows/watch.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Install dependencies
         run: pip install pyyaml
 
-      - name: Regenerate platform configs from live sources
+      - name: Regenerate platform configs from stable upstream sources
         run: |
           python3 -c "
           import sys, yaml
@@ -40,21 +40,28 @@ jobs:
               return dumper.represent_scalar('tag:yaml.org,2002:str', data)
           yaml.add_representer(str, str_representer)
 
-          # Regenerate retroarch.yml from live System.dat + core-info
-          from scraper.libretro_scraper import Scraper as LS
-          config = LS().generate_platform_yaml()
-          with open('platforms/retroarch.yml', 'w') as f:
-              yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
-          print(f'RetroArch: {len(config[\"systems\"])} systems, version={config[\"version\"]}')
-
-          # Regenerate batocera.yml from live batocera-systems
-          from scraper.batocera_scraper import Scraper as BS
-          config = BS().generate_platform_yaml()
-          with open('platforms/batocera.yml', 'w') as f:
-              yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
-          print(f'Batocera: {len(config[\"systems\"])} systems, version={config[\"version\"]}')
+          targets = {
+              'libretro': 'retroarch', 'batocera': 'batocera',
+              'recalbox': 'recalbox', 'retrobat': 'retrobat',
+              'romm': 'romm', 'bizhawk': 'bizhawk',
+              'retrodeck': 'retrodeck', 'emudeck': 'emudeck',
+          }
+          for scraper_name, out in targets.items():
+              mod = __import__(f'scraper.{scraper_name}_scraper', fromlist=['Scraper'])
+              config = mod.Scraper().generate_platform_yaml()
+              with open(f'platforms/{out}.yml', 'w') as f:
+                  yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+              print(f'{out}: {len(config[\"systems\"])} systems, version={config[\"version\"]}')
           "
 
+      - name: Refresh hardware target configs
+        run: |
+          python -m scripts.scraper.targets.batocera_targets_scraper -o platforms/targets/batocera.yml
+          python -m scripts.scraper.targets.retroarch_targets_scraper -o platforms/targets/retroarch.yml
+          python -m scripts.scraper.targets.emudeck_targets_scraper -o platforms/targets/emudeck.yml
+          python -m scripts.scraper.targets.retropie_targets_scraper -o platforms/targets/retropie.yml
+        continue-on-error: true
+
       - name: Auto-fetch missing BIOS files
         run: |
           python scripts/generate_db.py --bios-dir bios --output database.json

scripts/scraper/batocera_scraper.py

@@ -18,18 +18,57 @@ from pathlib import Path
 
 import yaml
 
-from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag
+from .base_scraper import BaseScraper, BiosRequirement
 
 PLATFORM_NAME = "batocera"
 
+_RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux"
+
+
+def pick_stable_tag(names: list[str]) -> str | None:
+    """Pick the highest batocera-N(.M) tag from a list of tag names."""
+    import re
+
+    stable = []
+    for name in names:
+        m = re.fullmatch(r"batocera-(\d+(?:\.\d+)?)", name)
+        if m:
+            stable.append((tuple(int(x) for x in m.group(1).split(".")), name))
+    return max(stable)[1] if stable else None
+
+
+def fetch_stable_tag() -> str | None:
+    """Return the newest stable batocera-N(.M) tag name."""
+    import json
+    import urllib.error
+    import urllib.request
+
+    url = "https://api.github.com/repos/batocera-linux/batocera.linux/tags?per_page=100"
+    try:
+        req = urllib.request.Request(
+            url,
+            headers={
+                "User-Agent": "retrobios-scraper/1.0",
+                "Accept": "application/vnd.github.v3+json",
+            },
+        )
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            tags = json.loads(resp.read())
+    except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError):
+        return None
+    return pick_stable_tag([tag["name"] for tag in tags])
+
+
+_STABLE_TAG = fetch_stable_tag() or "master"
+
 SOURCE_URL = (
-    "https://raw.githubusercontent.com/batocera-linux/batocera.linux/"
-    "master/package/batocera/core/batocera-scripts/scripts/batocera-systems"
+    f"{_RAW_BASE}/{_STABLE_TAG}"
+    "/package/batocera/core/batocera-scripts/scripts/batocera-systems"
 )
 
 CONFIGGEN_DEFAULTS_URL = (
-    "https://raw.githubusercontent.com/batocera-linux/batocera.linux/"
-    "master/package/batocera/core/batocera-configgen/configs/"
+    f"{_RAW_BASE}/{_STABLE_TAG}"
+    "/package/batocera/core/batocera-configgen/configs/"
     "configgen-defaults.yml"
 )
 
@@ -337,14 +376,9 @@ class Scraper(BaseScraper):
 
             systems[req.system]["files"].append(entry)
 
-        tag = fetch_github_latest_tag(
-            "batocera-linux/batocera.linux", prefix="batocera-"
-        )
         batocera_version = ""
-        if tag:
-            num = tag.removeprefix("batocera-")
-            if num.isdigit():
-                batocera_version = num
+        if _STABLE_TAG != "master":
+            batocera_version = _STABLE_TAG.removeprefix("batocera-")
         if not batocera_version:
             # Preserve existing version when fetch fails (offline mode)
             existing = (

scripts/scraper/bizhawk_scraper.py

@@ -24,26 +24,28 @@ try:
     from .base_scraper import (
         BaseScraper,
         BiosRequirement,
-        fetch_github_latest_tag,
+        fetch_github_latest_version,
         scraper_cli,
     )
 except ImportError:
     from base_scraper import (
         BaseScraper,
         BiosRequirement,
-        fetch_github_latest_tag,
+        fetch_github_latest_version,
         scraper_cli,
     )
 
 PLATFORM_NAME = "bizhawk"
 
-SOURCE_URL = (
-    "https://raw.githubusercontent.com/TASEmulators/BizHawk/"
-    "master/src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs"
-)
-
 GITHUB_REPO = "TASEmulators/BizHawk"
 
+_STABLE_TAG = fetch_github_latest_version(GITHUB_REPO) or "master"
+
+SOURCE_URL = (
+    f"https://raw.githubusercontent.com/TASEmulators/BizHawk/{_STABLE_TAG}"
+    "/src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs"
+)
+
 STATUS_RANK = {
     "Bad": 0,
     "Unacceptable": 1,
@@ -378,7 +380,7 @@ class Scraper(BaseScraper):
 
             systems[req.system]["files"].append(entry)
 
-        version = fetch_github_latest_tag(GITHUB_REPO) or ""
+        version = _STABLE_TAG if _STABLE_TAG != "master" else ""
 
         return {
             "platform": "BizHawk",

scripts/scraper/recalbox_scraper.py

@@ -18,12 +18,32 @@ from __future__ import annotations
 import sys
 import xml.etree.ElementTree as ET
 
-from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag
+from .base_scraper import BaseScraper, BiosRequirement
 
 PLATFORM_NAME = "recalbox"
 
+def _fetch_gitlab_stable_tag() -> str | None:
+    """Fetch the latest stable x.y.z tag from the Recalbox GitLab."""
+    import json
+    import re
+    import urllib.error
+    import urllib.request
+
+    url = "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tags?per_page=50"
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            tags = json.loads(resp.read())
+    except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError):
+        return None
+    stable = [t["name"] for t in tags if re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", t["name"])]
+    return stable[0] if stable else None
+
+
+_STABLE_TAG = _fetch_gitlab_stable_tag() or "master"
+
 SOURCE_URL = (
-    "https://gitlab.com/recalbox/recalbox/-/raw/master/"
+    f"https://gitlab.com/recalbox/recalbox/-/raw/{_STABLE_TAG}/"
     "board/recalbox/fsoverlay/recalbox/share_init/system/"
     ".emulationstation/es_bios.xml"
 )
@@ -225,8 +245,7 @@ class Scraper(BaseScraper):
 
             systems[req.system]["files"].append(entry)
 
-        version = fetch_github_latest_tag("recalbox/recalbox", prefix="") or ""
-        # Recalbox uses GitLab - GitHub API may not resolve
+        version = _STABLE_TAG if _STABLE_TAG != "master" else ""
         if not version:
             version = "10.0"
 

scripts/scraper/retrodeck_scraper.py

@@ -422,9 +422,15 @@ class Scraper(BaseScraper):
                 file_entry["md5"] = req.md5
             sys_entry["files"].append(file_entry)
 
+        try:
+            from .base_scraper import fetch_github_latest_version
+        except ImportError:
+            from scraper.base_scraper import fetch_github_latest_version
+        version = fetch_github_latest_version("RetroDECK/RetroDECK") or ""
+
         return {
             "platform": "RetroDECK",
-            "version": "",
+            "version": version,
             "homepage": "https://retrodeck.net",
             "source": "https://github.com/RetroDECK/components",
             "base_destination": "",
@@ -436,7 +442,10 @@ class Scraper(BaseScraper):
 
 
 def main() -> None:
-    from scraper.base_scraper import scraper_cli
+    try:
+        from .base_scraper import scraper_cli
+    except ImportError:
+        from scraper.base_scraper import scraper_cli
 
     scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements")
 

scripts/scraper/romm_scraper.py

@@ -32,13 +32,15 @@ except ImportError:
 
 PLATFORM_NAME = "romm"
 
-SOURCE_URL = (
-    "https://raw.githubusercontent.com/rommapp/romm/"
-    "master/backend/models/fixtures/known_bios_files.json"
-)
-
 GITHUB_REPO = "rommapp/romm"
 
+_STABLE_TAG = fetch_github_latest_version(GITHUB_REPO) or "master"
+
+SOURCE_URL = (
+    f"https://raw.githubusercontent.com/rommapp/romm/{_STABLE_TAG}"
+    "/backend/models/fixtures/known_bios_files.json"
+)
+
 # IGDB slug -> retrobios system ID
 SLUG_MAP: dict[str, str] = {
     "3do": "3do",
@@ -201,10 +203,7 @@ class Scraper(BaseScraper):
 
             systems[req.system]["files"].append(entry)
 
-        version = ""
-        tag = fetch_github_latest_version(GITHUB_REPO)
-        if tag:
-            version = tag
+        version = _STABLE_TAG if _STABLE_TAG != "master" else ""
 
         return {
             "inherits": "emulatorjs",

scripts/scraper/targets/batocera_targets_scraper.py

@@ -24,7 +24,13 @@ from . import BaseTargetScraper
 PLATFORM_NAME = "batocera"
 
 GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents"
-RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master"
+try:
+    from ..batocera_scraper import fetch_stable_tag
+except ImportError:
+    from scraper.batocera_scraper import fetch_stable_tag
+
+_STABLE_TAG = fetch_stable_tag() or "master"
+RAW_BASE = f"https://raw.githubusercontent.com/batocera-linux/batocera.linux/{_STABLE_TAG}"
 
 CONFIG_IN_URL = f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in"
 ES_SYSTEMS_URL = (