fix: send the token from the tag helper

scripts/scraper/base_scraper.py

@@ -390,24 +390,20 @@ def scraper_cli(
     print(f"Scraped {len(reqs)} requirements. Use --dry-run, --json, or --output.")
 
 
-def fetch_github_latest_tag(repo: str, prefix: str = "") -> str | None:
-    """Fetch the most recent matching tag from a GitHub repo."""
+def fetch_github_latest_tag(repo: str, prefix: str = "") -> str:
+    """Return the most recent tag of a GitHub repo, matching `prefix` if given.
+
+    Raises like fetch_github_latest_version: a tag that cannot be read is not
+    a reason to scrape a branch under it.
+    """
     url = f"https://api.github.com/repos/{repo}/tags?per_page=50"
     try:
-        req = urllib.request.Request(
-            url,
-            headers={
-                "User-Agent": "retrobios-scraper/1.0",
-                "Accept": "application/vnd.github.v3+json",
-            },
-        )
+        req = urllib.request.Request(url, headers=github_headers())
         with urllib.request.urlopen(req, timeout=15) as resp:
             tags = json.loads(resp.read())
-            for tag in tags:
-                name = tag["name"]
-                if prefix and not name.startswith(prefix):
-                    continue
-                return name
-            return tags[0]["name"] if tags else None
-    except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError):
-        return None
+    except (urllib.error.URLError, json.JSONDecodeError) as e:
+        raise RuntimeError(f"cannot read the tags of {repo}: {e}") from e
+    names = [tag["name"] for tag in tags if tag["name"].startswith(prefix)]
+    if not names:
+        raise RuntimeError(f"{repo} has no tag starting with {prefix!r}")
+    return names[0]

tests/test_scraper_contract.py

@@ -315,5 +315,48 @@ class EveryGitHubApiCallerSendsTheToken(unittest.TestCase):
         )
         self.assertEqual(silent, [])
 
+    def test_each_function_building_an_api_request_reads_the_token(self):
+        """Per module, one call of github_headers anywhere covered them all:
+        fetch_github_latest_tag sent no token beside a helper that did."""
+        import ast  # noqa: PLC0415
+
+        scripts = Path(__file__).resolve().parent.parent / "scripts"
+        silent = []
+        for path in sorted(scripts.rglob("*.py")):
+            source = path.read_text(encoding="utf-8")
+            for node in ast.walk(ast.parse(source)):
+                if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    continue
+                body = ast.get_source_segment(source, node) or ""
+                requests = any(
+                    isinstance(call, ast.Call)
+                    and getattr(call.func, "attr", getattr(call.func, "id", ""))
+                    in ("Request", "urlopen")
+                    for call in ast.walk(node)
+                )
+                if (
+                    requests
+                    and "api.github.com" in body
+                    and "github_headers(" not in body
+                    and "GITHUB_TOKEN" not in body
+                ):
+                    silent.append(f"{path.relative_to(scripts)}:{node.name}")
+        self.assertEqual(silent, [])
+
+    def test_the_tag_helper_sends_the_token_and_raises(self):
+        from scraper import base_scraper  # noqa: PLC0415
+
+        seen = {}
+
+        def answer(request, timeout=None):
+            seen.update(request.headers)
+            raise urllib.error.HTTPError(request.full_url, 403, "rate limited", {}, None)
+
+        with mock.patch.dict("os.environ", {"GITHUB_TOKEN": "t0k"}), mock.patch(
+            "urllib.request.urlopen", answer
+        ), self.assertRaises(RuntimeError):
+            base_scraper.fetch_github_latest_tag("o/r")
+        self.assertEqual(seen.get("Authorization"), "Bearer t0k")
+
 if __name__ == "__main__":
     unittest.main()

wiki/adding-a-scraper.md

@@ -96,6 +96,9 @@ Two additional functions in `base_scraper.py`:
 - `fetch_github_latest_tag(repo, prefix)` - fetches the most recent tag matching
   an optional prefix.
 
+Both send `GITHUB_TOKEN` when it is set and raise `RuntimeError` when the tag
+cannot be read, rather than letting a scraper fall back to a branch.
+
 ## Creating a BIOS scraper
 
 ### Minimal example