From 4544006affcf4124bfbeed9a97faf6c3beda5503 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:34:12 +0200 Subject: [PATCH] feat: wait out a forge rate limit while scraping --- scripts/scraper/base_scraper.py | 48 +++++++++++++++++++++----- scripts/scraper/recalbox_scraper.py | 16 ++------- tests/test_scraper_contract.py | 52 +++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py index 9b09cbba..4d5089d9 100644 --- a/scripts/scraper/base_scraper.py +++ b/scripts/scraper/base_scraper.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import os import sys +import time import urllib.error import urllib.request @@ -12,6 +13,8 @@ import yaml from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from typing import Callable + from common import write_text_atomic, yaml_load @@ -122,6 +125,40 @@ def _read_limited(resp: object, max_bytes: int = MAX_RESPONSE_SIZE) -> bytes: return b"".join(chunks) +# Statuses a forge answers when it asks the client to slow down. +_RETRYABLE = frozenset({429, 502, 503, 504}) + + +def fetch_bytes( + url: str, + headers: dict[str, str] | None = None, + attempts: int = 5, + sleep: Callable[[float], None] = time.sleep, +) -> bytes: + """GET a URL, waiting out a rate limit instead of failing the scrape. + + GitLab answers 429 after about a hundred raw-file reads, and Recalbox's + cores live in one system.ini per system: the scrape died midway. The + wait is the server's Retry-After when it gives one, else 2, 4, 8, 16 s. + """ + request = urllib.request.Request( + url, headers={"User-Agent": "retrobios-scraper/1.0", **(headers or {})} + ) + for attempt in range(attempts): + try: + with urllib.request.urlopen(request, timeout=30) as resp: + return _read_limited(resp) + except urllib.error.HTTPError as e: + if e.code not in _RETRYABLE or attempt == attempts - 1: + raise ConnectionError(f"Failed to fetch {url}: {e}") from e + retry_after = e.headers.get("Retry-After", "") if e.headers else "" + delay = float(retry_after) if retry_after.isdigit() else 2.0 ** (attempt + 1) + sleep(min(delay, 120.0)) + except urllib.error.URLError as e: + raise ConnectionError(f"Failed to fetch {url}: {e}") from e + raise AssertionError("unreachable") + + class BaseScraper(ABC): """Abstract base class for platform BIOS requirement scrapers.""" @@ -135,15 +172,8 @@ class BaseScraper(ABC): return self._raw_data if not self.url: raise ValueError("No source URL configured") - try: - req = urllib.request.Request( - self.url, headers={"User-Agent": "retrobios-scraper/1.0"} - ) - with urllib.request.urlopen(req, timeout=30) as resp: - self._raw_data = _read_limited(resp).decode("utf-8") - return self._raw_data - except urllib.error.URLError as e: - raise ConnectionError(f"Failed to fetch {self.url}: {e}") from e + self._raw_data = fetch_bytes(self.url).decode("utf-8") + return self._raw_data @abstractmethod def fetch_requirements(self) -> list[BiosRequirement]: diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index 0fb0bcd7..daeb61a2 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -17,7 +17,7 @@ from __future__ import annotations from common import parse_untrusted_xml -from .base_scraper import BaseScraper, BiosRequirement, requirement_entry +from .base_scraper import BaseScraper, BiosRequirement, fetch_bytes, requirement_entry PLATFORM_NAME = "recalbox" @@ -169,21 +169,11 @@ class Scraper(BaseScraper): systems ES lists and the cores a user can pick for each. """ import json - import urllib.error - import urllib.request - - def get(url: str) -> bytes: - req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return resp.read() - except urllib.error.URLError as e: - raise ConnectionError(f"Failed to fetch {url}: {e}") from e systems: list[str] = [] page = 1 while True: - listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page))) + listing = json.loads(fetch_bytes(SYSTEMS_TREE_URL.format(tag=self.tag, page=page))) systems += [ item["name"] for item in listing if item.get("type") == "tree" and not item["name"].startswith(".") @@ -196,7 +186,7 @@ class Scraper(BaseScraper): cores: set[str] = set() standalone: set[str] = set() for system in systems: - text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8") + text = fetch_bytes(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8") found, alone = system_ini_cores(text) cores |= found standalone |= alone diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index 471ca21f..2a5d226b 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -124,6 +124,58 @@ class TestConnection(unittest.TestCase): _StubScraper(error=KeyError("bug")).test_connection() +class RateLimitedForge(unittest.TestCase): + """GitLab answered 429 midway through Recalbox's system.ini files and + the whole scrape stopped.""" + + @staticmethod + def _limited(retry_after: str | None): + import email.message # noqa: PLC0415 + import io # noqa: PLC0415 + + headers = email.message.Message() + if retry_after is not None: + headers["Retry-After"] = retry_after + return urllib.error.HTTPError("u", 429, "Too Many Requests", headers, io.BytesIO()) + + def test_a_rate_limit_is_waited_out(self): + import io # noqa: PLC0415 + + from scraper.base_scraper import fetch_bytes # noqa: PLC0415 + + waits: list[float] = [] + answers = [self._limited("7"), self._limited(None), io.BytesIO(b"ok")] + + def urlopen(_request, timeout): + answer = answers.pop(0) + if isinstance(answer, Exception): + raise answer + return answer + + with mock.patch("urllib.request.urlopen", urlopen): + self.assertEqual(fetch_bytes("https://x", sleep=waits.append), b"ok") + self.assertEqual(waits, [7.0, 4.0]) + + def test_a_lasting_limit_or_a_missing_file_fails(self): + from scraper.base_scraper import fetch_bytes # noqa: PLC0415 + + def limited(_request, timeout): + raise self._limited(None) + + with mock.patch("urllib.request.urlopen", limited): + with self.assertRaises(ConnectionError): + fetch_bytes("https://x", attempts=3, sleep=lambda _s: None) + + def gone(_request, timeout): + raise urllib.error.HTTPError("u", 404, "Not Found", None, None) + + waits: list[float] = [] + with mock.patch("urllib.request.urlopen", gone): + with self.assertRaises(ConnectionError): + fetch_bytes("https://x", sleep=waits.append) + self.assertEqual(waits, []) + + class RetroPiePackageList(unittest.TestCase): """RetroPie states its packages, and nothing else states them for it."""