fix: send the github token on every api call

scripts/download.py

@@ -88,13 +88,16 @@ class Pack:
 def get_latest_release() -> dict:
     """Fetch latest release info from GitHub API."""
     url = f"{API}/repos/{REPO}/releases/latest"
-    req = urllib.request.Request(
+    headers = {
-        url,
+        "User-Agent": "retrobios-downloader/1.0",
-        headers={
+        "Accept": "application/vnd.github.v3+json",
-            "User-Agent": "retrobios-downloader/1.0",
+    }
-            "Accept": "application/vnd.github.v3+json",
+    # The anonymous quota is 60 calls an hour per address. The token goes to
-        },
+    # GitHub's API only: RETROBIOS_API can name any host.
-    )
+    token = os.environ.get("GITHUB_TOKEN")
+    if token and urllib.parse.urlsplit(url).hostname == "api.github.com":
+        headers["Authorization"] = f"Bearer {token}"
+    req = urllib.request.Request(url, headers=headers)
 
     try:
         with urllib.request.urlopen(req, timeout=30) as resp:

scripts/scraper/coreinfo_scraper.py

@@ -20,7 +20,12 @@ import urllib.error
 import urllib.request
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_github_latest_version,
+        github_headers,
+    )
 except ImportError:
     # Allow running directly: python scripts/scraper/coreinfo_scraper.py
     import os
@@ -30,6 +35,7 @@ except ImportError:
         BaseScraper,
         BiosRequirement,
         fetch_github_latest_version,
+        github_headers,
     )
 
 PLATFORM_NAME = "libretro_coreinfo"
@@ -209,13 +215,7 @@ class Scraper(BaseScraper):
         # Use the tree API to get all files at once
         url = f"{GITHUB_API}/git/trees/master?recursive=1"
         try:
-            req = urllib.request.Request(
+            req = urllib.request.Request(url, headers=github_headers())
-                url,
-                headers={
-                    "User-Agent": "retrobios-scraper/1.0",
-                    "Accept": "application/vnd.github.v3+json",
-                },
-            )
             with urllib.request.urlopen(req, timeout=30) as resp:
                 data = json.loads(resp.read())
 

scripts/scraper/romset_dat_importer.py

@@ -318,7 +318,9 @@ FBNEO_DATS_API = "https://api.github.com/repos/libretro/FBNeo/contents/dats"
 
 
 def _api_json(url: str) -> object:
-    request = urllib.request.Request(url, headers={"User-Agent": "retrobios"})
+    from .base_scraper import github_headers
+
+    request = urllib.request.Request(url, headers=github_headers())
     with urllib.request.urlopen(request, timeout=60) as response:
         return json.load(response)
 

tests/test_scraper_contract.py

@@ -285,5 +285,22 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
             libretro_scraper.Scraper()._fetch_core_metadata()
 
 
+
+class EveryGitHubApiCallerSendsTheToken(unittest.TestCase):
+    """romset_dat_importer and coreinfo_scraper called api.github.com with no
+    token: past the anonymous quota, --fetch failed with 403 while
+    refresh_stale had put a token in their environment."""
+
+    def test_each_module_naming_the_api_reads_the_token(self):
+        scripts = Path(__file__).resolve().parent.parent / "scripts"
+        silent = sorted(
+            str(path.relative_to(scripts))
+            for path in scripts.rglob("*.py")
+            if "api.github.com" in (source := path.read_text(encoding="utf-8"))
+            and "GITHUB_TOKEN" not in source
+            and "github_headers(" not in source
+        )
+        self.assertEqual(silent, [])
+
 if __name__ == "__main__":
     unittest.main()