libretro/scripts/restore_large_files.py

#!/usr/bin/env python3
"""Restore gitignored large files into the checkout from the release cache.

Files over 50 MB live as assets of the `large-files` release instead of in
git. A CI checkout is therefore incomplete: every consumer that resolves a
database entry against the disk (verify.py, generate_pack.py, and through
them generate_readme.py) reports those paths as missing.

Assets are matched by content, not by name: the SHA1 of each cached file is
looked up in the database, and the entry's path is written only when it is
gitignored and absent.

Usage:
    python scripts/restore_large_files.py [--cache .cache/large] [--db database.json]
"""

from __future__ import annotations

import argparse
import contextlib
import hashlib
import os
import sys
import tempfile

sys.path.insert(0, os.path.dirname(__file__))
from artifacts import set_plain_mode
from common import copy_file_atomic, load_database


def gitignored_paths(gitignore: str) -> set[str]:
    """Paths the repository keeps out of git, as written in .gitignore."""
    try:
        with open(gitignore) as f:
            return {
                line.strip() for line in f if line.strip().startswith("bios/")
            }
    except FileNotFoundError:
        return set()


def index_cache(cache_dir: str) -> dict[str, str]:
    """Map SHA1 to cached file path for every asset in the cache."""
    index: dict[str, str] = {}
    for name in sorted(os.listdir(cache_dir)):
        # A download in progress writes `<asset>.<random>.tmp` beside the
        # cache and renames it when done: it is partial, and may be gone by
        # the time it is opened.
        if name.endswith(".tmp"):
            continue
        path = os.path.join(cache_dir, name)
        if not os.path.isfile(path):
            continue
        digest = hashlib.sha1()
        with open(path, "rb") as f:
            for chunk in iter(lambda: f.read(1 << 20), b""):
                digest.update(chunk)
        index[digest.hexdigest()] = path
    return index


def _copy_verified(source: str, path: str, sha1: str) -> bool:
    """Copy *source* to *path* only if the copy hashes to *sha1*.

    The cache was hashed before the copy, and another run can replace a
    cached name in between (a re-uploaded asset): the copy is what counts.
    Whole or absent: a scan running beside a plain copy hashed the truncated
    file and published that in the database.
    """
    handle, scratch = tempfile.mkstemp(
        dir=os.path.dirname(os.path.abspath(path)),
        prefix=f".{os.path.basename(path)}.", suffix=".tmp",
    )
    os.close(handle)
    try:
        copy_file_atomic(source, scratch)
        digest = hashlib.sha1()
        with open(scratch, "rb") as f:
            for chunk in iter(lambda: f.read(1 << 20), b""):
                digest.update(chunk)
        if digest.hexdigest() != sha1:
            print(f"Changed in the cache while restoring: {source}", file=sys.stderr)
            return False
        set_plain_mode(scratch)
        os.replace(scratch, path)
        return True
    finally:
        with contextlib.suppress(FileNotFoundError):
            os.unlink(scratch)


def restore(
    cache_dir: str, db_path: str, gitignore: str
) -> tuple[int, list[str]]:
    ignored = gitignored_paths(gitignore)
    # No cache is not nothing to do: the paths it would have supplied are
    # still missing, and the check below is the one that must say so.
    if os.path.isdir(cache_dir):
        index = index_cache(cache_dir)
    else:
        print(f"No cache at {cache_dir}")
        index = {}
    db = load_database(db_path)
    restored = 0
    unsatisfied: list[str] = []
    for sha1, entry in db.get("files", {}).items():
        path = entry.get("path", "")
        if path not in ignored or os.path.exists(path):
            continue
        source = index.get(sha1)
        if not source:
            unsatisfied.append(path)
            continue
        os.makedirs(os.path.dirname(path), exist_ok=True)
        if not _copy_verified(source, path, sha1):
            unsatisfied.append(path)
            continue
        print(f"Restored: {path}")
        restored += 1
    print(f"Total: {restored} files restored")
    if unsatisfied:
        # Every consumer downstream resolves against the disk, so a path the
        # cache cannot supply is not a smaller restore: it drops entries from
        # the manifest and inflates the missing count the README publishes.
        print(f"Unsatisfied: {len(unsatisfied)} declared paths the cache "
              "cannot supply", file=sys.stderr)
        for path in sorted(unsatisfied)[:10]:
            print(f"  {path}", file=sys.stderr)
    return restored, unsatisfied


def main() -> None:
    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__)
    parser.add_argument(
        "--cache", default=".cache/large",
        help="where release assets are cached (default: %(default)s)",
    )
    parser.add_argument(
        "--db", default="database.json",
        help="database file to read (default: %(default)s)",
    )
    parser.add_argument(
        "--gitignore", default=".gitignore",
        help="the .gitignore listing release-asset files (default: %(default)s)",
    )
    args = parser.parse_args()
    _restored, unsatisfied = restore(args.cache, args.db, args.gitignore)
    if unsatisfied:
        raise SystemExit(1)


if __name__ == "__main__":
    from common import run_cli

    run_cli(main)