diff --git a/scripts/download.sh b/scripts/download.sh index c6e06e14..e2416779 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -66,7 +66,19 @@ normalize() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9' } +# The anonymous quota is 60 calls an hour per address. The token goes to +# GitHub's API only: RETROBIOS_API can name any host. The header is read +# from stdin so the token stays out of the process list. fetch_release() { + case "$API_BASE" in + https://api.github.com|https://api.github.com/) + if [ -n "${GITHUB_TOKEN:-}" ]; then + printf 'Authorization: Bearer %s\n' "$GITHUB_TOKEN" | + curl -fsSL -H @- "$API" + return + fi + ;; + esac curl -fsSL "$API" } diff --git a/tests/test_download.py b/tests/test_download.py index aa0ff85f..5c5c9927 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -632,5 +632,52 @@ class AProgressBarOnlyOnATerminal(unittest.TestCase): self.assertEqual(out.getvalue().count("\n"), 1) +class TheShellDownloaderSendsTheToken(unittest.TestCase): + """download.py sent GITHUB_TOKEN to the API and download.sh did not, so + past the anonymous quota the shell twin failed with 403 where the Python + one succeeded. A curl stand-in records what each call carries.""" + + def _calls(self, **env: str) -> str: + scratch = Path(_tempdir(self)) + log = scratch / "calls.log" + fake = scratch / "curl" + fake.write_text( + "#!/bin/sh\n" + f'printf "%s\\n" "$*" >> "{log}"\n' + 'case "$*" in *"-H @-"*) cat >> "' + str(log) + '" ;; esac\n' + "printf '{}'\n", + encoding="utf-8", + ) + fake.chmod(0o755) + base = {k: v for k, v in os.environ.items() + if k not in ("GITHUB_TOKEN", "RETROBIOS_API")} + proc = subprocess.run( + ["bash", str(SHELL), "--list"], + env={**base, "PATH": f"{scratch}{os.pathsep}{os.environ['PATH']}", **env}, + capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + return log.read_text(encoding="utf-8") + + def test_the_api_call_carries_the_token(self): + calls = self._calls(GITHUB_TOKEN="sekrit") + self.assertIn("Authorization: Bearer sekrit", calls) + self.assertIn("api.github.com/repos/", calls) + + def test_the_token_stays_out_of_the_command_line(self): + calls = self._calls(GITHUB_TOKEN="sekrit") + command = calls.splitlines()[0] + self.assertNotIn("sekrit", command) + + def test_another_host_never_receives_it(self): + calls = self._calls( + GITHUB_TOKEN="sekrit", RETROBIOS_API="https://mirror.example" + ) + self.assertNotIn("sekrit", calls) + + def test_no_token_sends_no_header(self): + self.assertNotIn("Authorization", self._calls()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index 6df589b5..19e37d69 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -657,10 +657,11 @@ class EveryGitHubApiCallerSendsTheToken(unittest.TestCase): refresh_stale had put a token in their environment.""" def test_each_module_naming_the_api_reads_the_token(self): + # download.sh is download.py's twin and called the API bare. scripts = Path(__file__).resolve().parent.parent / "scripts" silent = sorted( str(path.relative_to(scripts)) - for path in scripts.rglob("*.py") + for path in [*scripts.rglob("*.py"), *scripts.rglob("*.sh")] if "api.github.com" in (source := path.read_text(encoding="utf-8")) and "GITHUB_TOKEN" not in source and "github_headers(" not in source