feat: pass GITHUB_TOKEN to declared API steps

A step marked github_token: true receives the token. autorun accepts the key only on check_freshness.py, refresh_stale.py and python -m scripts.scraper.* tracked in this repository, refuses it on any other step and in the campaign job. The nightly steps declare it.

automation/jobs.yml

@@ -6,8 +6,10 @@
 # commits. `python` names the interpreter running autorun.py. A step is a
 # command list, or a mapping with `run`, `ok_codes`, `stdout` (a file the
 # command's output is written to) and `env` (variables passed through
-# beside PATH, HOME, LANG, LC_*, TZ and TMPDIR). A step never sees a token
-# or a password: only the push and gh receive GITHUB_TOKEN. `twice`
+# beside PATH, HOME, LANG, LC_*, TZ and TMPDIR). A step never sees a password.
+# GITHUB_TOKEN goes to the push, to gh, and to a step marked `github_token:
+# true`, which autorun accepts only on check_freshness.py, refresh_stale.py and
+# `-m scripts.scraper.*`, tracked here, never in the campaign job. `twice`
 # recomputes an artifact the gate wrote into `{out}` and compares both,
 # timestamps aside. Each commit
 # takes only the changed files its `paths` globs match, and a job that
@@ -47,10 +49,14 @@ jobs:
       - run: [python, scripts/check_freshness.py, --json]
         ok_codes: [0, 1]
         stdout: tmp/autorun/freshness.json
-      - [python, scripts/refresh_stale.py, --findings, tmp/autorun/freshness.json,
-         --only, "data,catalogs"]
-      - [python, -m, scripts.scraper.mame_hash_scraper]
-      - [python, -m, scripts.scraper.fbneo_hash_scraper]
+        github_token: true
+      - run: [python, scripts/refresh_stale.py, --findings, tmp/autorun/freshness.json,
+              --only, "data,catalogs"]
+        github_token: true
+      - run: [python, -m, scripts.scraper.mame_hash_scraper]
+        github_token: true
+      - run: [python, -m, scripts.scraper.fbneo_hash_scraper]
+        github_token: true
     gate:
       - [python, scripts/pipeline.py, --offline]
     twice:

scripts/autorun.py

@@ -102,6 +102,8 @@ class Step:
     stdout: str = ""
     # Variables passed through beside the allowlist (romfetch's settings).
     env: tuple[str, ...] = ()
+    # GITHUB_TOKEN for a first-party script that calls the GitHub API.
+    github_token: bool = False
 
 
 @dataclass(frozen=True)
@@ -194,22 +196,61 @@ def check_subject(subject: str, body: str = "") -> list[str]:
     return problems(f"{subject}\n\n{body}" if body else subject)
 
 
-def _step(raw: object, where: str) -> Step:
+# The first-party scripts that call the GitHub API and may receive
+# GITHUB_TOKEN: run as `python <script>` or `python -m <module>`, and tracked
+# by git in this repository. Nothing else ever gets it, third-party code least.
+TOKEN_SCRIPTS = ("scripts/check_freshness.py", "scripts/refresh_stale.py")
+TOKEN_MODULE_PREFIX = "scripts.scraper."
+NO_TOKEN_JOBS = frozenset({"campaign"})
+
+
+def tracked(path: str, repo: Path = REPO_ROOT) -> bool:
+    proc = subprocess.run(["git", "-C", str(repo), "ls-files", "--error-unmatch", "--", path],
+                          capture_output=True, check=False)
+    return proc.returncode == 0
+
+
+def token_script(argv: Sequence[str], repo: Path = REPO_ROOT) -> str:
+    """The tracked first-party script a step runs, or '' when it runs anything else."""
+    if len(argv) < 2 or argv[0] != "python":
+        return ""
+    if argv[1] in TOKEN_SCRIPTS:
+        path = argv[1]
+    elif argv[1] == "-m" and len(argv) > 2 and argv[2].startswith(TOKEN_MODULE_PREFIX) \
+            and re.fullmatch(r"[a-z0-9_.]+", argv[2]):
+        path = argv[2].replace(".", "/") + ".py"
+    else:
+        return ""
+    return path if tracked(path, repo) else ""
+
+
+def _step(raw: object, where: str, job: str = "") -> Step:
     if isinstance(raw, list):
         return Step(argv=tuple(str(a) for a in raw))
     if isinstance(raw, dict) and isinstance(raw.get("run"), list):
-        unknown = set(raw) - {"run", "ok_codes", "stdout", "env"}
+        unknown = set(raw) - {"run", "ok_codes", "stdout", "env", "github_token"}
         if unknown:
             raise ValueError(f"{where}: unknown step key {sorted(unknown)}")
         names = tuple(str(n) for n in raw.get("env") or ())
         secret = [n for n in names if is_secret(n)]
         if secret:
             raise ValueError(f"{where}: a step never receives {', '.join(secret)}")
+        argv = tuple(str(a) for a in raw["run"])
+        token = raw.get("github_token", False)
+        if token is not False:
+            if token is not True:
+                raise ValueError(f"{where}: github_token is true or absent")
+            if job in NO_TOKEN_JOBS:
+                raise ValueError(f"{where}: job {job} never receives GITHUB_TOKEN")
+            if not token_script(argv):
+                raise ValueError(f"{where}: github_token only goes to {', '.join(TOKEN_SCRIPTS)} "
+                                 f"or python -m {TOKEN_MODULE_PREFIX}*, tracked in this repository")
         return Step(
-            argv=tuple(str(a) for a in raw["run"]),
+            argv=argv,
             ok_codes=tuple(int(c) for c in raw.get("ok_codes", (0,))),
             stdout=str(raw.get("stdout") or ""),
             env=names,
+            github_token=token is True,
         )
     raise ValueError(f"{where}: a step is a command list or a mapping with 'run'")
 
@@ -244,8 +285,8 @@ def parse_jobs(data: dict) -> dict[str, Job]:
             raise ValueError(f"job {name}: a job that commits runs in the worktree")
         jobs[name] = Job(
             name=name,
-            steps=tuple(_step(s, f"job {name}") for s in raw.get("steps") or ()),
-            gate=tuple(_step(s, f"job {name} gate") for s in raw.get("gate") or ()),
+            steps=tuple(_step(s, f"job {name}", name) for s in raw.get("steps") or ()),
+            gate=tuple(_step(s, f"job {name} gate", name) for s in raw.get("gate") or ()),
             twice=tuple(twice),
             commits=tuple(commits),
             schedule=str(raw.get("schedule") or ""),
@@ -748,12 +789,18 @@ def token_env() -> dict[str, str]:
 
 def job_env(job: Job, step: Step | None = None) -> dict[str, str]:
     """A step's environment: the allowlist, the runner's own markers and the
-    variables the step names. Never a token, network job or not."""
+    variables the step names. GITHUB_TOKEN only for a step that declares
+    github_token, which the registry allows on first-party scripts alone."""
     add = {"RETROBIOS_AUTO": "1"}
     if job.lock:
         # A step that takes the host lock itself (campaign.py) runs under ours.
         add[LOCK_HELD_ENV] = "1"
-    return safe_env(step.env if step else (), add)
+    env = safe_env(step.env if step else (), add)
+    if step is not None and step.github_token and job.name not in NO_TOKEN_JOBS:
+        token = token_env().get("GITHUB_TOKEN")
+        if token:
+            env["GITHUB_TOKEN"] = token
+    return env
 
 
 def _remove_scratch(root: Path, rel: str) -> None:

tests/test_autorun.py

@@ -236,6 +236,48 @@ class StepEnvironment(Scratch):
         self.assertEqual(given["GITHUB_TOKEN"], "t1")
 
 
+class GithubTokenPerStep(Scratch):
+    """GITHUB_TOKEN goes to first-party scripts that call the API, declared
+    per step, and never to third-party code or the campaign."""
+
+    def job(self, run: list[str], name: str = "x") -> dict:
+        return ar.parse_jobs({"jobs": {name: {"steps": [{"run": run, "github_token": True}]}}})
+
+    def test_allowed_on_the_scrapers_and_freshness_scripts(self):
+        for run in ([PY, "-m", "scripts.scraper.mame_hash_scraper"],
+                    [PY, "scripts/check_freshness.py", "--json"],
+                    [PY, "scripts/refresh_stale.py", "--only", "data"]):
+            with self.subTest(run=run):
+                step = self.job(run)["x"].steps[0]
+                self.assertTrue(step.github_token)
+                with unittest.mock.patch.object(ar, "token_env", return_value={"GITHUB_TOKEN": "t1"}):
+                    self.assertEqual(ar.job_env(self.job(run)["x"], step)["GITHUB_TOKEN"], "t1")
+
+    def test_refused_on_any_other_step(self):
+        for run in ([PY, "-c", "print(1)"], [PY, "scripts/campaign.py", "run"],
+                    [PY, "scripts/core_probe.py", "run"], ["sh", "-c", "env"],
+                    [PY, "scripts/not_tracked_here.py"], [PY, "-m", "scripts.scraper.not_tracked"],
+                    [PY, "-m", "os"], ["/usr/bin/python3", "scripts/check_freshness.py"]):
+            with self.subTest(run=run), self.assertRaises(ValueError):
+                self.job(run)
+
+    def test_refused_in_the_campaign_job_even_on_a_scraper(self):
+        with self.assertRaisesRegex(ValueError, "campaign never receives"):
+            self.job([PY, "-m", "scripts.scraper.mame_hash_scraper"], name="campaign")
+
+    def test_an_undeclared_step_gets_no_token(self):
+        job = ar.parse_jobs({"jobs": {"x": {"network": True, "steps": [
+            [PY, "-m", "scripts.scraper.mame_hash_scraper"]]}}})["x"]
+        with unittest.mock.patch.object(ar, "token_env", return_value={"GITHUB_TOKEN": "t1"}), \
+                unittest.mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t1"}):
+            self.assertNotIn("GITHUB_TOKEN", ar.job_env(job, job.steps[0]))
+
+    def test_the_committed_registry(self):
+        jobs = ar.load_jobs()
+        self.assertTrue(all(step.github_token for step in jobs["nightly"].steps))
+        self.assertFalse(any(step.github_token for step in jobs["campaign"].steps + jobs["campaign"].gate))
+
+
 class ReportAlert(Scratch):
     def test_a_report_alert_is_raised_then_cleared(self):
         write = "import json, sys; json.dump({'alert': sys.argv[1]} if sys.argv[1] else {}, open('r.json', 'w'))"

wiki/tools.md

@@ -717,7 +717,7 @@ them is a reviewed commit.
 | Load | Waits while the 1-minute load is above a quarter of the CPUs, up to `max_load_wait` seconds, then `deferred`. |
 | Disk | Below the job's `min_free_gb` the job is `refused`. Under 100 GB free the `disk` alert is raised. |
 | Retry | Exit 139 (or signal 11), `SystemError` or "unknown opcode" is run again once on an idle host. A second failure is a failure. |
-| Environment | A step, gate or recomputation inherits `PATH`, `HOME`, `LANG`, `LC_*`, `TZ` and `TMPDIR`, the runner's own markers and the variables its `env:` key names. A name with `TOKEN`, `SECRET` or `PASS` in it, `GH_TOKEN` and the RomStation account are dropped even when named, and a step naming one is refused when the registry loads. Only the push and the `gh` calls that read CI get `GITHUB_TOKEN`. |
+| Environment | A step, gate or recomputation inherits `PATH`, `HOME`, `LANG`, `LC_*`, `TZ` and `TMPDIR`, the runner's own markers and the variables its `env:` key names. A name with `TOKEN`, `SECRET` or `PASS` in it, `GH_TOKEN` and the RomStation account are dropped even when named, and a step naming one is refused when the registry loads. `GITHUB_TOKEN` goes to the push, the `gh` calls that read CI, and a step marked `github_token: true`. The registry accepts that key only on `python scripts/check_freshness.py`, `python scripts/refresh_stale.py` and `python -m scripts.scraper.*` tracked in this repository, and never in the `campaign` job. |
 | Double computation | Each `twice` entry recomputes an artifact the gate wrote and compares both, timestamps aside. A difference is recomputed once on an idle host, then the job fails with the `host-integrity` alert. |
 
 Jobs that commit run in `.cache/autorun/worktree`, on the branch `autorun`,