diff --git a/.github/workflows/watch.yml b/.github/workflows/watch.yml index 987a3a0a..42afb181 100644 --- a/.github/workflows/watch.yml +++ b/.github/workflows/watch.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: pip install pyyaml - - name: Regenerate platform configs from live sources + - name: Regenerate platform configs from stable upstream sources run: | python3 -c " import sys, yaml @@ -40,21 +40,28 @@ jobs: return dumper.represent_scalar('tag:yaml.org,2002:str', data) yaml.add_representer(str, str_representer) - # Regenerate retroarch.yml from live System.dat + core-info - from scraper.libretro_scraper import Scraper as LS - config = LS().generate_platform_yaml() - with open('platforms/retroarch.yml', 'w') as f: - yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False) - print(f'RetroArch: {len(config[\"systems\"])} systems, version={config[\"version\"]}') - - # Regenerate batocera.yml from live batocera-systems - from scraper.batocera_scraper import Scraper as BS - config = BS().generate_platform_yaml() - with open('platforms/batocera.yml', 'w') as f: - yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False) - print(f'Batocera: {len(config[\"systems\"])} systems, version={config[\"version\"]}') + targets = { + 'libretro': 'retroarch', 'batocera': 'batocera', + 'recalbox': 'recalbox', 'retrobat': 'retrobat', + 'romm': 'romm', 'bizhawk': 'bizhawk', + 'retrodeck': 'retrodeck', 'emudeck': 'emudeck', + } + for scraper_name, out in targets.items(): + mod = __import__(f'scraper.{scraper_name}_scraper', fromlist=['Scraper']) + config = mod.Scraper().generate_platform_yaml() + with open(f'platforms/{out}.yml', 'w') as f: + yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False) + print(f'{out}: {len(config[\"systems\"])} systems, version={config[\"version\"]}') " + - name: Refresh hardware target configs + run: | + python -m scripts.scraper.targets.batocera_targets_scraper -o platforms/targets/batocera.yml + python -m scripts.scraper.targets.retroarch_targets_scraper -o platforms/targets/retroarch.yml + python -m scripts.scraper.targets.emudeck_targets_scraper -o platforms/targets/emudeck.yml + python -m scripts.scraper.targets.retropie_targets_scraper -o platforms/targets/retropie.yml + continue-on-error: true + - name: Auto-fetch missing BIOS files run: | python scripts/generate_db.py --bios-dir bios --output database.json diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index cc9cc184..291c8a5a 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -18,18 +18,57 @@ from pathlib import Path import yaml -from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag +from .base_scraper import BaseScraper, BiosRequirement PLATFORM_NAME = "batocera" +_RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux" + + +def pick_stable_tag(names: list[str]) -> str | None: + """Pick the highest batocera-N(.M) tag from a list of tag names.""" + import re + + stable = [] + for name in names: + m = re.fullmatch(r"batocera-(\d+(?:\.\d+)?)", name) + if m: + stable.append((tuple(int(x) for x in m.group(1).split(".")), name)) + return max(stable)[1] if stable else None + + +def fetch_stable_tag() -> str | None: + """Return the newest stable batocera-N(.M) tag name.""" + import json + import urllib.error + import urllib.request + + url = "https://api.github.com/repos/batocera-linux/batocera.linux/tags?per_page=100" + try: + req = urllib.request.Request( + url, + headers={ + "User-Agent": "retrobios-scraper/1.0", + "Accept": "application/vnd.github.v3+json", + }, + ) + with urllib.request.urlopen(req, timeout=15) as resp: + tags = json.loads(resp.read()) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): + return None + return pick_stable_tag([tag["name"] for tag in tags]) + + +_STABLE_TAG = fetch_stable_tag() or "master" + SOURCE_URL = ( - "https://raw.githubusercontent.com/batocera-linux/batocera.linux/" - "master/package/batocera/core/batocera-scripts/scripts/batocera-systems" + f"{_RAW_BASE}/{_STABLE_TAG}" + "/package/batocera/core/batocera-scripts/scripts/batocera-systems" ) CONFIGGEN_DEFAULTS_URL = ( - "https://raw.githubusercontent.com/batocera-linux/batocera.linux/" - "master/package/batocera/core/batocera-configgen/configs/" + f"{_RAW_BASE}/{_STABLE_TAG}" + "/package/batocera/core/batocera-configgen/configs/" "configgen-defaults.yml" ) @@ -337,14 +376,9 @@ class Scraper(BaseScraper): systems[req.system]["files"].append(entry) - tag = fetch_github_latest_tag( - "batocera-linux/batocera.linux", prefix="batocera-" - ) batocera_version = "" - if tag: - num = tag.removeprefix("batocera-") - if num.isdigit(): - batocera_version = num + if _STABLE_TAG != "master": + batocera_version = _STABLE_TAG.removeprefix("batocera-") if not batocera_version: # Preserve existing version when fetch fails (offline mode) existing = ( diff --git a/scripts/scraper/bizhawk_scraper.py b/scripts/scraper/bizhawk_scraper.py index 406400a2..2a0d314a 100644 --- a/scripts/scraper/bizhawk_scraper.py +++ b/scripts/scraper/bizhawk_scraper.py @@ -24,26 +24,28 @@ try: from .base_scraper import ( BaseScraper, BiosRequirement, - fetch_github_latest_tag, + fetch_github_latest_version, scraper_cli, ) except ImportError: from base_scraper import ( BaseScraper, BiosRequirement, - fetch_github_latest_tag, + fetch_github_latest_version, scraper_cli, ) PLATFORM_NAME = "bizhawk" -SOURCE_URL = ( - "https://raw.githubusercontent.com/TASEmulators/BizHawk/" - "master/src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs" -) - GITHUB_REPO = "TASEmulators/BizHawk" +_STABLE_TAG = fetch_github_latest_version(GITHUB_REPO) or "master" + +SOURCE_URL = ( + f"https://raw.githubusercontent.com/TASEmulators/BizHawk/{_STABLE_TAG}" + "/src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs" +) + STATUS_RANK = { "Bad": 0, "Unacceptable": 1, @@ -378,7 +380,7 @@ class Scraper(BaseScraper): systems[req.system]["files"].append(entry) - version = fetch_github_latest_tag(GITHUB_REPO) or "" + version = _STABLE_TAG if _STABLE_TAG != "master" else "" return { "platform": "BizHawk", diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index f3ff0540..259762fa 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -18,12 +18,32 @@ from __future__ import annotations import sys import xml.etree.ElementTree as ET -from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag +from .base_scraper import BaseScraper, BiosRequirement PLATFORM_NAME = "recalbox" +def _fetch_gitlab_stable_tag() -> str | None: + """Fetch the latest stable x.y.z tag from the Recalbox GitLab.""" + import json + import re + import urllib.error + import urllib.request + + url = "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tags?per_page=50" + try: + req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + tags = json.loads(resp.read()) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): + return None + stable = [t["name"] for t in tags if re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", t["name"])] + return stable[0] if stable else None + + +_STABLE_TAG = _fetch_gitlab_stable_tag() or "master" + SOURCE_URL = ( - "https://gitlab.com/recalbox/recalbox/-/raw/master/" + f"https://gitlab.com/recalbox/recalbox/-/raw/{_STABLE_TAG}/" "board/recalbox/fsoverlay/recalbox/share_init/system/" ".emulationstation/es_bios.xml" ) @@ -225,8 +245,7 @@ class Scraper(BaseScraper): systems[req.system]["files"].append(entry) - version = fetch_github_latest_tag("recalbox/recalbox", prefix="") or "" - # Recalbox uses GitLab - GitHub API may not resolve + version = _STABLE_TAG if _STABLE_TAG != "master" else "" if not version: version = "10.0" diff --git a/scripts/scraper/retrodeck_scraper.py b/scripts/scraper/retrodeck_scraper.py index ecfbba15..881b9a58 100644 --- a/scripts/scraper/retrodeck_scraper.py +++ b/scripts/scraper/retrodeck_scraper.py @@ -422,9 +422,15 @@ class Scraper(BaseScraper): file_entry["md5"] = req.md5 sys_entry["files"].append(file_entry) + try: + from .base_scraper import fetch_github_latest_version + except ImportError: + from scraper.base_scraper import fetch_github_latest_version + version = fetch_github_latest_version("RetroDECK/RetroDECK") or "" + return { "platform": "RetroDECK", - "version": "", + "version": version, "homepage": "https://retrodeck.net", "source": "https://github.com/RetroDECK/components", "base_destination": "", @@ -436,7 +442,10 @@ class Scraper(BaseScraper): def main() -> None: - from scraper.base_scraper import scraper_cli + try: + from .base_scraper import scraper_cli + except ImportError: + from scraper.base_scraper import scraper_cli scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements") diff --git a/scripts/scraper/romm_scraper.py b/scripts/scraper/romm_scraper.py index ae9c5df3..b6fc1da8 100644 --- a/scripts/scraper/romm_scraper.py +++ b/scripts/scraper/romm_scraper.py @@ -32,13 +32,15 @@ except ImportError: PLATFORM_NAME = "romm" -SOURCE_URL = ( - "https://raw.githubusercontent.com/rommapp/romm/" - "master/backend/models/fixtures/known_bios_files.json" -) - GITHUB_REPO = "rommapp/romm" +_STABLE_TAG = fetch_github_latest_version(GITHUB_REPO) or "master" + +SOURCE_URL = ( + f"https://raw.githubusercontent.com/rommapp/romm/{_STABLE_TAG}" + "/backend/models/fixtures/known_bios_files.json" +) + # IGDB slug -> retrobios system ID SLUG_MAP: dict[str, str] = { "3do": "3do", @@ -201,10 +203,7 @@ class Scraper(BaseScraper): systems[req.system]["files"].append(entry) - version = "" - tag = fetch_github_latest_version(GITHUB_REPO) - if tag: - version = tag + version = _STABLE_TAG if _STABLE_TAG != "master" else "" return { "inherits": "emulatorjs", diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index d30d2f67..91dd0b7e 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -24,7 +24,13 @@ from . import BaseTargetScraper PLATFORM_NAME = "batocera" GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents" -RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master" +try: + from ..batocera_scraper import fetch_stable_tag +except ImportError: + from scraper.batocera_scraper import fetch_stable_tag + +_STABLE_TAG = fetch_stable_tag() or "master" +RAW_BASE = f"https://raw.githubusercontent.com/batocera-linux/batocera.linux/{_STABLE_TAG}" CONFIG_IN_URL = f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in" ES_SYSTEMS_URL = (