feat: read recalbox system files in batches

scripts/scraper/base_scraper.py

@@ -134,6 +134,7 @@ def fetch_bytes(
     headers: dict[str, str] | None = None,
     attempts: int = 5,
     sleep: Callable[[float], None] = time.sleep,
+    data: bytes | None = None,
 ) -> bytes:
     """GET a URL, waiting out a rate limit instead of failing the scrape.
 
@@ -142,7 +143,7 @@ def fetch_bytes(
     wait is the server's Retry-After when it gives one, else 2, 4, 8, 16 s.
     """
     request = urllib.request.Request(
-        url, headers={"User-Agent": "retrobios-scraper/1.0", **(headers or {})}
+        url, data=data, headers={"User-Agent": "retrobios-scraper/1.0", **(headers or {})}
     )
     for attempt in range(attempts):
         try:

scripts/scraper/recalbox_scraper.py

@@ -25,15 +25,11 @@ def _fetch_gitlab_stable_tag() -> str:
     """Fetch the latest stable x.y.z tag from the Recalbox GitLab."""
     import json
     import re
-    import urllib.error
-    import urllib.request
 
     url = "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tags?per_page=50"
     try:
-        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
-        with urllib.request.urlopen(req, timeout=15) as resp:
-            tags = json.loads(resp.read())
-    except (urllib.error.URLError, json.JSONDecodeError) as e:
+        tags = json.loads(fetch_bytes(url))
+    except (ConnectionError, json.JSONDecodeError) as e:
         raise RuntimeError(f"cannot list Recalbox tags: {e}") from e
     stable = [t["name"] for t in tags if re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", t["name"])]
     if not stable:
@@ -105,10 +101,15 @@ SYSTEMS_TREE_URL = (
     "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree"
     "?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}"
 )
-SYSTEM_INI_URL = (
-    "https://gitlab.com/recalbox/recalbox/-/raw/{tag}"
-    "/package/recalbox-romfs2/systems/{system}/system.ini"
+SYSTEMS_DIR = "package/recalbox-romfs2/systems"
+# One raw read per system.ini made about 150 requests and GitLab throttled
+# the scrape past half an hour; GraphQL returns a batch of blobs at once.
+GRAPHQL_URL = "https://gitlab.com/api/graphql"
+BLOBS_QUERY = (
+    "query($paths: [String!]!, $ref: String!) { project(fullPath: \"recalbox/recalbox\") "
+    "{ repository { blobs(paths: $paths, ref: $ref) { nodes { path rawBlob } } } } }"
 )
+BLOBS_PER_QUERY = 50
 
 
 def system_ini_cores(text: str) -> tuple[set[str], set[str]]:
@@ -162,6 +163,29 @@ class Scraper(BaseScraper):
         self.tag = _fetch_gitlab_stable_tag()
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
+    def _system_inis(self, systems: list[str]) -> list[str]:
+        """Every system's system.ini at the tag, read in batches."""
+        import json
+
+        paths = [f"{SYSTEMS_DIR}/{system}/system.ini" for system in systems]
+        texts: dict[str, str] = {}
+        for start in range(0, len(paths), BLOBS_PER_QUERY):
+            batch = paths[start:start + BLOBS_PER_QUERY]
+            body = json.dumps({"query": BLOBS_QUERY,
+                               "variables": {"paths": batch, "ref": self.tag}}).encode()
+            answer = json.loads(fetch_bytes(
+                GRAPHQL_URL, data=body, headers={"Content-Type": "application/json"},
+            ))
+            try:
+                nodes = answer["data"]["project"]["repository"]["blobs"]["nodes"]
+            except (KeyError, TypeError) as exc:
+                raise RuntimeError(f"GitLab returned no blobs: {answer.get('errors')}") from exc
+            texts.update({n["path"]: n["rawBlob"] for n in nodes if n.get("rawBlob")})
+        missing = [path for path in paths if path not in texts]
+        if missing:
+            raise RuntimeError(f"no system.ini at {self.tag} for {', '.join(missing[:5])}")
+        return [texts[path] for path in paths]
+
     def _fetch_cores(self) -> tuple[list[str], list[str]]:
         """Every core Recalbox's systems offer, and those it runs standalone.
 
@@ -185,8 +209,7 @@ class Scraper(BaseScraper):
             raise RuntimeError(f"no Recalbox system found at {self.tag}")
         cores: set[str] = set()
         standalone: set[str] = set()
-        for system in systems:
-            text = fetch_bytes(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
+        for text in self._system_inis(systems):
             found, alone = system_ini_cores(text)
             cores |= found
             standalone |= alone

tests/test_scraper_contract.py

@@ -176,6 +176,38 @@ class RateLimitedForge(unittest.TestCase):
         self.assertEqual(waits, [])
 
 
+class RecalboxReadsItsSystemsInBatches(unittest.TestCase):
+    """One raw read per system.ini, about 150, kept GitLab throttling the
+    scrape past its half-hour timeout."""
+
+    def test_one_query_per_fifty_systems(self):
+        import json as _json  # noqa: PLC0415
+
+        from scraper import recalbox_scraper  # noqa: PLC0415
+
+        systems = [f"sys{i:03}" for i in range(120)]
+        queries: list[list[str]] = []
+
+        def fetch(url, data=None, headers=None):
+            if "repository/tree" in url:
+                page = int(url.rsplit("page=", 1)[1])
+                names = systems[(page - 1) * 100:page * 100]
+                return _json.dumps([{"name": n, "type": "tree"} for n in names]).encode()
+            paths = _json.loads(data)["variables"]["paths"]
+            queries.append(paths)
+            nodes = [{"path": p, "rawBlob": "[core.0]\nemulator = libretro\ncore = "
+                      + p.split("/")[-2] + "\n"} for p in paths]
+            return _json.dumps({"data": {"project": {"repository": {
+                "blobs": {"nodes": nodes}}}}}).encode()
+
+        scraper = recalbox_scraper.Scraper.__new__(recalbox_scraper.Scraper)
+        scraper.tag = "10.1.1"
+        with mock.patch.object(recalbox_scraper, "fetch_bytes", fetch):
+            cores, _standalone = scraper._fetch_cores()
+        self.assertEqual([len(q) for q in queries], [50, 50, 20])
+        self.assertEqual(len(cores), 120)
+
+
 class RetroPiePackageList(unittest.TestCase):
     """RetroPie states its packages, and nothing else states them for it."""