feat: wait out a forge rate limit while scraping

scripts/scraper/base_scraper.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import os
 import sys
+import time
 import urllib.error
 import urllib.request
 
@@ -12,6 +13,8 @@ import yaml
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Callable
+
 from common import write_text_atomic, yaml_load
 
 
@@ -122,6 +125,40 @@ def _read_limited(resp: object, max_bytes: int = MAX_RESPONSE_SIZE) -> bytes:
     return b"".join(chunks)
 
 
+# Statuses a forge answers when it asks the client to slow down.
+_RETRYABLE = frozenset({429, 502, 503, 504})
+
+
+def fetch_bytes(
+    url: str,
+    headers: dict[str, str] | None = None,
+    attempts: int = 5,
+    sleep: Callable[[float], None] = time.sleep,
+) -> bytes:
+    """GET a URL, waiting out a rate limit instead of failing the scrape.
+
+    GitLab answers 429 after about a hundred raw-file reads, and Recalbox's
+    cores live in one system.ini per system: the scrape died midway. The
+    wait is the server's Retry-After when it gives one, else 2, 4, 8, 16 s.
+    """
+    request = urllib.request.Request(
+        url, headers={"User-Agent": "retrobios-scraper/1.0", **(headers or {})}
+    )
+    for attempt in range(attempts):
+        try:
+            with urllib.request.urlopen(request, timeout=30) as resp:
+                return _read_limited(resp)
+        except urllib.error.HTTPError as e:
+            if e.code not in _RETRYABLE or attempt == attempts - 1:
+                raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+            retry_after = e.headers.get("Retry-After", "") if e.headers else ""
+            delay = float(retry_after) if retry_after.isdigit() else 2.0 ** (attempt + 1)
+            sleep(min(delay, 120.0))
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+    raise AssertionError("unreachable")
+
+
 class BaseScraper(ABC):
     """Abstract base class for platform BIOS requirement scrapers."""
 
@@ -135,15 +172,8 @@ class BaseScraper(ABC):
             return self._raw_data
         if not self.url:
             raise ValueError("No source URL configured")
-        try:
-            req = urllib.request.Request(
-                self.url, headers={"User-Agent": "retrobios-scraper/1.0"}
-            )
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                self._raw_data = _read_limited(resp).decode("utf-8")
-                return self._raw_data
-        except urllib.error.URLError as e:
-            raise ConnectionError(f"Failed to fetch {self.url}: {e}") from e
+        self._raw_data = fetch_bytes(self.url).decode("utf-8")
+        return self._raw_data
 
     @abstractmethod
     def fetch_requirements(self) -> list[BiosRequirement]:

scripts/scraper/recalbox_scraper.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 
 from common import parse_untrusted_xml
 
-from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
+from .base_scraper import BaseScraper, BiosRequirement, fetch_bytes, requirement_entry
 
 PLATFORM_NAME = "recalbox"
 
@@ -169,21 +169,11 @@ class Scraper(BaseScraper):
         systems ES lists and the cores a user can pick for each.
         """
         import json
-        import urllib.error
-        import urllib.request
-
-        def get(url: str) -> bytes:
-            req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
-            try:
-                with urllib.request.urlopen(req, timeout=30) as resp:
-                    return resp.read()
-            except urllib.error.URLError as e:
-                raise ConnectionError(f"Failed to fetch {url}: {e}") from e
 
         systems: list[str] = []
         page = 1
         while True:
-            listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page)))
+            listing = json.loads(fetch_bytes(SYSTEMS_TREE_URL.format(tag=self.tag, page=page)))
             systems += [
                 item["name"] for item in listing
                 if item.get("type") == "tree" and not item["name"].startswith(".")
@@ -196,7 +186,7 @@ class Scraper(BaseScraper):
         cores: set[str] = set()
         standalone: set[str] = set()
         for system in systems:
-            text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
+            text = fetch_bytes(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
             found, alone = system_ini_cores(text)
             cores |= found
             standalone |= alone

tests/test_scraper_contract.py

@@ -124,6 +124,58 @@ class TestConnection(unittest.TestCase):
             _StubScraper(error=KeyError("bug")).test_connection()
 
 
+class RateLimitedForge(unittest.TestCase):
+    """GitLab answered 429 midway through Recalbox's system.ini files and
+    the whole scrape stopped."""
+
+    @staticmethod
+    def _limited(retry_after: str | None):
+        import email.message  # noqa: PLC0415
+        import io  # noqa: PLC0415
+
+        headers = email.message.Message()
+        if retry_after is not None:
+            headers["Retry-After"] = retry_after
+        return urllib.error.HTTPError("u", 429, "Too Many Requests", headers, io.BytesIO())
+
+    def test_a_rate_limit_is_waited_out(self):
+        import io  # noqa: PLC0415
+
+        from scraper.base_scraper import fetch_bytes  # noqa: PLC0415
+
+        waits: list[float] = []
+        answers = [self._limited("7"), self._limited(None), io.BytesIO(b"ok")]
+
+        def urlopen(_request, timeout):
+            answer = answers.pop(0)
+            if isinstance(answer, Exception):
+                raise answer
+            return answer
+
+        with mock.patch("urllib.request.urlopen", urlopen):
+            self.assertEqual(fetch_bytes("https://x", sleep=waits.append), b"ok")
+        self.assertEqual(waits, [7.0, 4.0])
+
+    def test_a_lasting_limit_or_a_missing_file_fails(self):
+        from scraper.base_scraper import fetch_bytes  # noqa: PLC0415
+
+        def limited(_request, timeout):
+            raise self._limited(None)
+
+        with mock.patch("urllib.request.urlopen", limited):
+            with self.assertRaises(ConnectionError):
+                fetch_bytes("https://x", attempts=3, sleep=lambda _s: None)
+
+        def gone(_request, timeout):
+            raise urllib.error.HTTPError("u", 404, "Not Found", None, None)
+
+        waits: list[float] = []
+        with mock.patch("urllib.request.urlopen", gone):
+            with self.assertRaises(ConnectionError):
+                fetch_bytes("https://x", sleep=waits.append)
+        self.assertEqual(waits, [])
+
+
 class RetroPiePackageList(unittest.TestCase):
     """RetroPie states its packages, and nothing else states them for it."""