diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index e29fd73b..f033fbf1 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -27,25 +27,16 @@ PLATFORM_NAME = "batocera" GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents" try: + from ..base_scraper import github_headers from ..batocera_scraper import fetch_stable_tag except ImportError: + from scraper.base_scraper import github_headers from scraper.batocera_scraper import fetch_stable_tag -_STABLE_TAG = fetch_stable_tag() or "master" -RAW_BASE = f"https://raw.githubusercontent.com/batocera-linux/batocera.linux/{_STABLE_TAG}" +RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/{tag}" -CONFIG_IN_URL = f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in" -ES_SYSTEMS_URL = ( - f"{RAW_BASE}/package/batocera/emulationstation/batocera-es-system/es_systems.yml" -) - -_HEADERS = { - "User-Agent": "retrobios-scraper/1.0", - "Accept": "application/vnd.github.v3+json", -} -import os as _os -if _os.environ.get("GITHUB_TOKEN"): - _HEADERS["Authorization"] = f"Bearer {_os.environ['GITHUB_TOKEN']}" +CONFIG_IN_PATH = "package/batocera/core/batocera-system/Config.in" +ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml" _TARGET_FLAG_RE = re.compile(r"^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y", re.MULTILINE) @@ -68,26 +59,22 @@ _META_BLOCK_RE = re.compile( ) -def _fetch(url: str, headers: dict | None = None) -> str | None: +def _fetch(url: str, headers: dict | None = None) -> str: h = headers or {"User-Agent": "retrobios-scraper/1.0"} try: req = urllib.request.Request(url, headers=h) with urllib.request.urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8") except urllib.error.URLError as e: - print(f" skip {url}: {e}", file=sys.stderr) - return None + raise RuntimeError(f"cannot fetch {url}: {e}") from e -def _fetch_json(url: str) -> list | dict | None: - text = _fetch(url, headers=_HEADERS) - if text is None: - return None +def _fetch_json(url: str) -> list | dict: + text = _fetch(url, headers=github_headers()) try: return json.loads(text) except json.JSONDecodeError as e: - print(f" json parse error {url}: {e}", file=sys.stderr) - return None + raise RuntimeError(f"cannot parse {url}: {e}") from e def _normalise_condition(raw: str) -> str: @@ -272,40 +259,41 @@ class Scraper(BaseTargetScraper): def __init__(self, url: str = "https://github.com/batocera-linux/batocera.linux"): super().__init__(url=url) + self.tag = "" + + def _raw(self, path: str) -> str: + return f"{RAW_BASE.format(tag=self.tag)}/{path}" def _list_boards(self) -> list[str]: """List batocera-*.board files from configs/ via GitHub API.""" - data = _fetch_json(f"{GITHUB_API}/configs") - if not data or not isinstance(data, list): - return [] - return [ + data = _fetch_json(f"{GITHUB_API}/configs?ref={self.tag}") + boards = [ item["name"] for item in data if isinstance(item, dict) and item.get("name", "").startswith("batocera-") and item.get("name", "").endswith(".board") - ] + ] if isinstance(data, list) else [] + if not boards: + raise RuntimeError(f"no batocera-*.board file in configs/ at {self.tag}") + return boards def _fetch_board_flag(self, board_name: str) -> str | None: """Fetch a board file and extract its BR2_PACKAGE_BATOCERA_TARGET_* flag.""" - url = f"{RAW_BASE}/configs/{board_name}" - text = _fetch(url) - if text is None: - return None - m = _TARGET_FLAG_RE.search(text) + m = _TARGET_FLAG_RE.search(_fetch(self._raw(f"configs/{board_name}"))) return m.group(1) if m else None def fetch_targets(self) -> dict: """Build per-board emulator availability map.""" + self.tag = fetch_stable_tag() print(" fetching board list...", file=sys.stderr) boards = self._list_boards() - if not boards: - print(" warning: no boards found", file=sys.stderr) print(" fetching Config.in...", file=sys.stderr) - config_in_text = _fetch(CONFIG_IN_URL) + config_in_url = self._raw(CONFIG_IN_PATH) + config_in_text = _fetch(config_in_url) if not config_in_text: - raise RuntimeError(f"empty Config.in from {CONFIG_IN_URL}") + raise RuntimeError(f"empty Config.in from {config_in_url}") meta_rules = _parse_meta_flags(config_in_text) selects = _parse_selects(config_in_text) @@ -314,19 +302,20 @@ class Scraper(BaseTargetScraper): file=sys.stderr, ) if not selects: - raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {CONFIG_IN_URL}") + raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {config_in_url}") print(" fetching es_systems.yml...", file=sys.stderr) - es_text = _fetch(ES_SYSTEMS_URL) + es_systems_url = self._raw(ES_SYSTEMS_PATH) + es_text = _fetch(es_systems_url) if not es_text: - raise RuntimeError(f"empty es_systems.yml from {ES_SYSTEMS_URL}") + raise RuntimeError(f"empty es_systems.yml from {es_systems_url}") package_to_emulators = _parse_es_systems(es_text) print( f" parsed {len(package_to_emulators)} package->emulator mappings", file=sys.stderr, ) if not package_to_emulators: - raise RuntimeError(f"no package->emulator mappings parsed from {ES_SYSTEMS_URL}") + raise RuntimeError(f"no package->emulator mappings parsed from {es_systems_url}") targets: dict[str, dict] = {} for board_name in sorted(boards): diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index d61d3ec5..13c6dd56 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -19,6 +19,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT)) from scripts.scraper.targets import ( # noqa: E402 + batocera_targets_scraper, emudeck_targets_scraper, retroarch_targets_scraper, retropie_targets_scraper, @@ -32,6 +33,7 @@ def _refuse(*_args, **_kwargs): class FailedRequestsStopTheScrape(unittest.TestCase): def test_every_scraper_raises(self): for module in ( + batocera_targets_scraper, emudeck_targets_scraper, retropie_targets_scraper, retroarch_targets_scraper, @@ -58,6 +60,58 @@ class FailedRequestsStopTheScrape(unittest.TestCase): ), self.assertRaises(RuntimeError): module.Scraper().fetch_targets() + def test_batocera_needs_its_boards(self): + """No board listed, or one board unreadable, used to print a warning + and write the targets file without those boards.""" + + class _Body: + def __init__(self, body: bytes): + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self.body + + def serve(boards: list[str], refuse_board: bool): + def urlopen(request, *_a, **_k): + url = request.full_url + if "/tags" in url: + return _Body(b'[{"name": "batocera-43"}]') + if "/contents/configs" in url: + return _Body(json.dumps([{"name": b} for b in boards]).encode()) + if url.endswith(".board"): + if refuse_board: + _refuse() + return _Body(b"BR2_PACKAGE_BATOCERA_TARGET_X86_64=y\n") + if url.endswith("Config.in"): + return _Body(b"config X\n\tselect BR2_PACKAGE_MGBA\n") + return _Body( + b"gba:\n emulators:\n libretro:\n" + b" mgba: {requireAnyOf: [BR2_PACKAGE_MGBA]}\n" + ) + + return urlopen + + for boards, refuse_board in (([], False), (["batocera-x86_64.board"], True)): + with self.subTest(boards=boards), mock.patch.object( + batocera_targets_scraper.urllib.request, + "urlopen", + serve(boards, refuse_board), + ), self.assertRaises(RuntimeError): + batocera_targets_scraper.Scraper().fetch_targets() + + with mock.patch.object( + batocera_targets_scraper.urllib.request, + "urlopen", + serve(["batocera-x86_64.board"], False), + ): + targets = batocera_targets_scraper.Scraper().fetch_targets()["targets"] + self.assertEqual(targets["x86_64"]["cores"], ["mgba"]) class RetroPieModuleFlags(unittest.TestCase):