From 3f029578afda7d3a9eb2c7b5014374278a03ce33 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Tue, 20 Oct 2026 04:36:24 +0200
Subject: [PATCH] fix: a scraper that cannot start says what to do

Scrapers resolve their tag or branch head in the constructor, which ran outside the try in scraper_cli: a refused GitHub call ended in a traceback. The error is now printed with the next step, setting GITHUB_TOKEN when the API refused an anonymous call.
---
 scripts/scraper/base_scraper.py | 29 +++++++++++++++++-----
 tests/test_audit_regressions.py | 43 +++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py
index e4703ff9..25112559 100644
--- a/scripts/scraper/base_scraper.py
+++ b/scripts/scraper/base_scraper.py
@@ -14,7 +14,7 @@ import yaml
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Callable
+from typing import Callable, NoReturn
 
 from common import write_text_atomic, yaml_load
 
@@ -334,6 +334,21 @@ def _quoted_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode:
 _PlatformDumper.add_representer(str, _quoted_str)
 
 
+def _exit_on_scrape_error(error: Exception) -> NoReturn:
+    """Print what failed and what to try, then exit 1 without a traceback."""
+    print(f"Error: {error}", file=sys.stderr)
+    refused = re.search(r"\b(403|429)\b|rate limit", str(error), re.I)
+    if refused and not os.environ.get("GITHUB_TOKEN"):
+        print(
+            "Anonymous GitHub API calls are limited to 60 an hour. "
+            "Run again with GITHUB_TOKEN=$(gh auth token) set.",
+            file=sys.stderr,
+        )
+    else:
+        print("Check the network and the URL above, then run again.", file=sys.stderr)
+    sys.exit(1)
+
+
 def scraper_cli(
     scraper_class: type, description: str = "Scrape BIOS requirements"
 ) -> None:
@@ -346,12 +361,11 @@ def scraper_cli(
     parser.add_argument("--json", action="store_true", help="Output as JSON")
     args = parser.parse_args()
 
-    scraper = scraper_class()
     try:
+        scraper = scraper_class()
         reqs = scraper.fetch_requirements()
-    except (ConnectionError, ValueError) as e:
-        print(f"Error: {e}", file=sys.stderr)
-        sys.exit(1)
+    except (OSError, ValueError, RuntimeError) as e:
+        _exit_on_scrape_error(e)
 
     if args.dry_run:
         by_system: dict[str, list] = {}
@@ -385,7 +399,10 @@ def scraper_cli(
         # Use scraper's generate_platform_yaml() if available (includes
         # platform metadata, cores list, standalone_cores, etc.)
         if hasattr(scraper, "generate_platform_yaml"):
-            config = scraper.generate_platform_yaml()
+            try:
+                config = scraper.generate_platform_yaml()
+            except (OSError, ValueError, RuntimeError) as e:
+                _exit_on_scrape_error(e)
         else:
             # Generic fallback: just systems from requirements
             config = {"systems": {}}
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index b979f3af..3f54e45b 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5483,5 +5483,48 @@ class DocsDescribeTheRegistriesTheyName(unittest.TestCase):
         self.assertIn("NODEDUP_DIRS", readme_gen)
 
 
+class AScraperThatCannotStartSaysWhy(unittest.TestCase):
+    """Scrapers resolve their tag or branch head in the constructor, and
+    scraper_cli built the scraper outside its try: a refused API call ended
+    in a traceback instead of a message saying to set GITHUB_TOKEN."""
+
+    def _run(self, scraper_class, token: str | None):
+        import contextlib
+        import io
+        from unittest import mock
+
+        from scraper.base_scraper import scraper_cli
+
+        env = {k: v for k, v in os.environ.items() if k != "GITHUB_TOKEN"}
+        if token:
+            env["GITHUB_TOKEN"] = token
+        err = io.StringIO()
+        with mock.patch.dict(os.environ, env, clear=True), \
+                mock.patch.object(sys, "argv", ["x", "--dry-run"]), \
+                contextlib.redirect_stderr(err), \
+                self.assertRaises(SystemExit) as stop:
+            scraper_cli(scraper_class)
+        return stop.exception.code, err.getvalue()
+
+    def test_a_failing_constructor_exits_with_a_hint(self):
+        class Refused:
+            def __init__(self):
+                raise RuntimeError("cannot read the head of a/b@main: HTTP 403")
+
+        code, err = self._run(Refused, None)
+        self.assertEqual(code, 1)
+        self.assertIn("HTTP 403", err)
+        self.assertIn("GITHUB_TOKEN", err)
+
+    def test_with_a_token_the_hint_points_elsewhere(self):
+        class Offline:
+            def fetch_requirements(self):
+                raise ConnectionError("no route to host")
+
+        code, err = self._run(Offline, "t")
+        self.assertEqual(code, 1)
+        self.assertNotIn("GITHUB_TOKEN", err)
+
+
 if __name__ == "__main__":
     unittest.main()