diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index ed259741..74e33a80 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -41,18 +41,20 @@ def pick_stable_tag(names: list[str]) -> str | None: def fetch_stable_tag() -> str | None: """Return the newest stable batocera-N(.M) tag name.""" import json + import os import urllib.error import urllib.request url = "https://api.github.com/repos/batocera-linux/batocera.linux/tags?per_page=100" + headers = { + "User-Agent": "retrobios-scraper/1.0", + "Accept": "application/vnd.github.v3+json", + } + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" try: - req = urllib.request.Request( - url, - headers={ - "User-Agent": "retrobios-scraper/1.0", - "Accept": "application/vnd.github.v3+json", - }, - ) + req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=15) as resp: tags = json.loads(resp.read()) except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index 59007fce..e29fd73b 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -43,6 +43,9 @@ _HEADERS = { "User-Agent": "retrobios-scraper/1.0", "Accept": "application/vnd.github.v3+json", } +import os as _os +if _os.environ.get("GITHUB_TOKEN"): + _HEADERS["Authorization"] = f"Bearer {_os.environ['GITHUB_TOKEN']}" _TARGET_FLAG_RE = re.compile(r"^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y", re.MULTILINE) diff --git a/scripts/scraper/targets/emudeck_targets_scraper.py b/scripts/scraper/targets/emudeck_targets_scraper.py index 96c61a88..a1d2b6df 100644 --- a/scripts/scraper/targets/emudeck_targets_scraper.py +++ b/scripts/scraper/targets/emudeck_targets_scraper.py @@ -9,6 +9,7 @@ from __future__ import annotations import argparse import json +import os import re import sys import urllib.error @@ -49,10 +50,12 @@ _SKIP = {"retroarch_maincfg", "retroarch"} def _fetch(url: str) -> str | None: + headers = {"User-Agent": "retrobios-scraper/1.0"} + token = os.environ.get("GITHUB_TOKEN") + if token and "api.github.com" in url: + headers["Authorization"] = f"Bearer {token}" try: - req = urllib.request.Request( - url, headers={"User-Agent": "retrobios-scraper/1.0"} - ) + req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8") except urllib.error.URLError as e: diff --git a/scripts/scraper/targets/retropie_targets_scraper.py b/scripts/scraper/targets/retropie_targets_scraper.py index 5b965bea..c266d69a 100644 --- a/scripts/scraper/targets/retropie_targets_scraper.py +++ b/scripts/scraper/targets/retropie_targets_scraper.py @@ -9,6 +9,7 @@ from __future__ import annotations import argparse import json +import os import re import sys import urllib.error @@ -59,11 +60,12 @@ _MODULE_FLAGS_RE = re.compile(r'rp_module_flags\s*=\s*["\']([^"\']*)["\']') def _fetch(url: str, accept: str = "text/plain") -> str | None: + headers = {"User-Agent": "retrobios-scraper/1.0", "Accept": accept} + token = os.environ.get("GITHUB_TOKEN") + if token and "api.github.com" in url: + headers["Authorization"] = f"Bearer {token}" try: - req = urllib.request.Request( - url, - headers={"User-Agent": "retrobios-scraper/1.0", "Accept": accept}, - ) + req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8") except urllib.error.URLError as e: