From d588ebbde46ef8aaa4a5d141e824a7f4c1d74301 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:34:27 +0200 Subject: [PATCH] chore: restore large files before ci coverage checks A CI checkout omits every file over 50 MB, so verify and generate_pack resolve those database entries against a disk that does not hold them and report them missing. The generated README then stops matching the committed one for a reason that has nothing to do with staleness. restore_large_files.py writes them back from the release cache, matched by SHA1 rather than by name, and only where the path is gitignored and absent. The site workflow runs it, and refreshes the data directories, before generating. --- .github/workflows/build.yml | 27 +--------- .github/workflows/deploy-site.yml | 14 +++++ scripts/restore_large_files.py | 88 +++++++++++++++++++++++++++++++ tests/test_audit_regressions.py | 79 +++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 scripts/restore_large_files.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f78d82ce..2c285f42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,32 +58,7 @@ jobs: run: | mkdir -p .cache/large gh release download large-files -D .cache/large/ 2>/dev/null || true - python3 -c " - import hashlib, json, os, shutil - db = json.load(open('database.json')) - with open('.gitignore') as f: - ignored = {l.strip() for l in f if l.strip().startswith('bios/')} - cache = '.cache/large' - if not os.path.isdir(cache): - exit(0) - idx = {} - for fn in os.listdir(cache): - fp = os.path.join(cache, fn) - if os.path.isfile(fp): - h = hashlib.sha1(open(fp, 'rb').read()).hexdigest() - idx[h] = fp - restored = 0 - for sha1, entry in db['files'].items(): - path = entry['path'] - if path in ignored and not os.path.exists(path): - src = idx.get(sha1) - if src: - os.makedirs(os.path.dirname(path), exist_ok=True) - shutil.copy2(src, path) - print(f'Restored: {path}') - restored += 1 - print(f'Total: {restored} files restored') - " + python scripts/restore_large_files.py env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 026795d2..1d22a071 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -52,6 +52,20 @@ jobs: - name: Validate data contracts run: python scripts/validate_schemas.py + # Coverage is resolved against the disk, so a checkout without the + # release assets and the data caches reports files as missing and the + # generated README stops matching the committed one. + - name: Restore large files from release + run: | + mkdir -p .cache/large + gh release download large-files -D .cache/large/ 2>/dev/null || true + python scripts/restore_large_files.py + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Refresh data directories + run: python scripts/refresh_data_dirs.py + - name: Generate site run: | python scripts/generate_site.py diff --git a/scripts/restore_large_files.py b/scripts/restore_large_files.py new file mode 100644 index 00000000..0945a39d --- /dev/null +++ b/scripts/restore_large_files.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Restore gitignored large files into the checkout from the release cache. + +Files over 50 MB live as assets of the `large-files` release instead of in +git. A CI checkout is therefore incomplete: every consumer that resolves a +database entry against the disk (verify.py, generate_pack.py, and through +them generate_readme.py) reports those paths as missing. + +Assets are matched by content, not by name: the SHA1 of each cached file is +looked up in the database, and the entry's path is written only when it is +gitignored and absent. + +Usage: + python scripts/restore_large_files.py [--cache .cache/large] [--db database.json] +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import shutil +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from common import load_database + + +def gitignored_paths(gitignore: str) -> set[str]: + """Paths the repository keeps out of git, as written in .gitignore.""" + try: + with open(gitignore) as f: + return { + line.strip() for line in f if line.strip().startswith("bios/") + } + except FileNotFoundError: + return set() + + +def index_cache(cache_dir: str) -> dict[str, str]: + """Map SHA1 to cached file path for every asset in the cache.""" + index: dict[str, str] = {} + for name in sorted(os.listdir(cache_dir)): + path = os.path.join(cache_dir, name) + if not os.path.isfile(path): + continue + digest = hashlib.sha1() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + index[digest.hexdigest()] = path + return index + + +def restore(cache_dir: str, db_path: str, gitignore: str) -> int: + if not os.path.isdir(cache_dir): + print(f"No cache at {cache_dir}, nothing to restore") + return 0 + ignored = gitignored_paths(gitignore) + index = index_cache(cache_dir) + db = load_database(db_path) + restored = 0 + for sha1, entry in db.get("files", {}).items(): + path = entry.get("path", "") + if path not in ignored or os.path.exists(path): + continue + source = index.get(sha1) + if not source: + continue + os.makedirs(os.path.dirname(path), exist_ok=True) + shutil.copy2(source, path) + print(f"Restored: {path}") + restored += 1 + print(f"Total: {restored} files restored") + return restored + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", default=".cache/large") + parser.add_argument("--db", default="database.json") + parser.add_argument("--gitignore", default=".gitignore") + args = parser.parse_args() + restore(args.cache, args.db, args.gitignore) + + +if __name__ == "__main__": + main() diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index aa1173f1..4fe50632 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -700,5 +700,84 @@ class TargetManifestRegressions(unittest.TestCase): generate_target_manifests(str(source), str(output)) +class CheckoutCompletenessRegressions(unittest.TestCase): + """Coverage is resolved against the disk, so the checkout must be whole. + + Files over 50 MB and the data directory caches are gitignored. A job that + regenerates the README or the site without restoring them counts those + files as missing and publishes a coverage the collection does not have. + """ + + def _steps(self, workflow: str, job: str) -> list[dict]: + data = yaml.safe_load((ROOT / ".github" / "workflows" / workflow).read_text()) + return data["jobs"][job]["steps"] + + def _index(self, steps: list[dict], needle: str) -> int: + for position, step in enumerate(steps): + if needle in step.get("name", "") or needle in str(step.get("run", "")): + return position + self.fail(f"no step matching {needle!r}") + + def test_site_deploy_completes_the_checkout_before_generating(self): + steps = self._steps("deploy-site.yml", "build") + generate = self._index(steps, "Generate site") + self.assertLess(self._index(steps, "restore_large_files.py"), generate) + self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate) + + def test_pack_build_completes_the_checkout_before_building(self): + steps = self._steps("build.yml", "release") + build = self._index(steps, "Build packs") + self.assertLess(self._index(steps, "restore_large_files.py"), build) + self.assertLess(self._index(steps, "refresh_data_dirs.py"), build) + + def test_restore_matches_assets_by_content_not_by_name(self): + from scripts.restore_large_files import restore + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + payload = b"firmware bytes" + (cache / "renamed-asset.bin").write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + (root / ".gitignore").write_text("bios/Sony/big.pup\n", encoding="utf-8") + (root / "database.json").write_text( + json.dumps({"files": {sha1: {"path": "bios/Sony/big.pup"}}}), + encoding="utf-8", + ) + cwd = os.getcwd() + os.chdir(root) + try: + self.assertEqual(restore(str(cache), "database.json", ".gitignore"), 1) + self.assertEqual((root / "bios/Sony/big.pup").read_bytes(), payload) + # A path already in the checkout is never overwritten. + self.assertEqual(restore(str(cache), "database.json", ".gitignore"), 0) + finally: + os.chdir(cwd) + + def test_restore_leaves_tracked_paths_alone(self): + from scripts.restore_large_files import restore + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + payload = b"tracked bytes" + (cache / "asset.bin").write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + (root / ".gitignore").write_text("bios/other.bin\n", encoding="utf-8") + (root / "database.json").write_text( + json.dumps({"files": {sha1: {"path": "bios/tracked.bin"}}}), + encoding="utf-8", + ) + cwd = os.getcwd() + os.chdir(root) + try: + self.assertEqual(restore(str(cache), "database.json", ".gitignore"), 0) + self.assertFalse((root / "bios/tracked.bin").exists()) + finally: + os.chdir(cwd) + + if __name__ == "__main__": unittest.main()