diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py index 290d983a..9b09cbba 100644 --- a/scripts/scraper/base_scraper.py +++ b/scripts/scraper/base_scraper.py @@ -390,24 +390,20 @@ def scraper_cli( print(f"Scraped {len(reqs)} requirements. Use --dry-run, --json, or --output.") -def fetch_github_latest_tag(repo: str, prefix: str = "") -> str | None: - """Fetch the most recent matching tag from a GitHub repo.""" +def fetch_github_latest_tag(repo: str, prefix: str = "") -> str: + """Return the most recent tag of a GitHub repo, matching `prefix` if given. + + Raises like fetch_github_latest_version: a tag that cannot be read is not + a reason to scrape a branch under it. + """ url = f"https://api.github.com/repos/{repo}/tags?per_page=50" try: - req = urllib.request.Request( - url, - headers={ - "User-Agent": "retrobios-scraper/1.0", - "Accept": "application/vnd.github.v3+json", - }, - ) + req = urllib.request.Request(url, headers=github_headers()) with urllib.request.urlopen(req, timeout=15) as resp: tags = json.loads(resp.read()) - for tag in tags: - name = tag["name"] - if prefix and not name.startswith(prefix): - continue - return name - return tags[0]["name"] if tags else None - except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): - return None + except (urllib.error.URLError, json.JSONDecodeError) as e: + raise RuntimeError(f"cannot read the tags of {repo}: {e}") from e + names = [tag["name"] for tag in tags if tag["name"].startswith(prefix)] + if not names: + raise RuntimeError(f"{repo} has no tag starting with {prefix!r}") + return names[0] diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index 304e64a4..d2f2fad4 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -315,5 +315,48 @@ class EveryGitHubApiCallerSendsTheToken(unittest.TestCase): ) self.assertEqual(silent, []) + def test_each_function_building_an_api_request_reads_the_token(self): + """Per module, one call of github_headers anywhere covered them all: + fetch_github_latest_tag sent no token beside a helper that did.""" + import ast # noqa: PLC0415 + + scripts = Path(__file__).resolve().parent.parent / "scripts" + silent = [] + for path in sorted(scripts.rglob("*.py")): + source = path.read_text(encoding="utf-8") + for node in ast.walk(ast.parse(source)): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + body = ast.get_source_segment(source, node) or "" + requests = any( + isinstance(call, ast.Call) + and getattr(call.func, "attr", getattr(call.func, "id", "")) + in ("Request", "urlopen") + for call in ast.walk(node) + ) + if ( + requests + and "api.github.com" in body + and "github_headers(" not in body + and "GITHUB_TOKEN" not in body + ): + silent.append(f"{path.relative_to(scripts)}:{node.name}") + self.assertEqual(silent, []) + + def test_the_tag_helper_sends_the_token_and_raises(self): + from scraper import base_scraper # noqa: PLC0415 + + seen = {} + + def answer(request, timeout=None): + seen.update(request.headers) + raise urllib.error.HTTPError(request.full_url, 403, "rate limited", {}, None) + + with mock.patch.dict("os.environ", {"GITHUB_TOKEN": "t0k"}), mock.patch( + "urllib.request.urlopen", answer + ), self.assertRaises(RuntimeError): + base_scraper.fetch_github_latest_tag("o/r") + self.assertEqual(seen.get("Authorization"), "Bearer t0k") + if __name__ == "__main__": unittest.main() diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index 2727624e..d0888470 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -96,6 +96,9 @@ Two additional functions in `base_scraper.py`: - `fetch_github_latest_tag(repo, prefix)` - fetches the most recent tag matching an optional prefix. +Both send `GITHUB_TOKEN` when it is set and raise `RuntimeError` when the tag +cannot be read, rather than letting a scraper fall back to a branch. + ## Creating a BIOS scraper ### Minimal example