refactor: build cores from an exported tree

make runs on a git archive export of the commit and its submodules, the clone and its .git never bound into the sandbox. Host-side git runs with no user or system config, no hooks, no fsmonitor and the file and ext transports refused.

scripts/campaign.py

@@ -44,6 +44,7 @@ import argparse
 import contextlib
 import datetime as dt
 import hashlib
+import io
 import json
 import os
 import re
@@ -51,6 +52,7 @@ import shutil
 import signal
 import subprocess
 import sys
+import tarfile
 import time
 import urllib.error
 import urllib.request
@@ -282,6 +284,55 @@ def child_env(romfetch: bool = False) -> dict[str, str]:
     return autorun.safe_env(names)
 
 
+# Host-side git on a clone of a third-party project: no hook, no fsmonitor,
+# no local-file transport, no user or system config.
+GIT_SAFE = ("-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false",
+            "-c", "protocol.file.allow=never", "-c", "protocol.ext.allow=never")
+
+
+def host_git(*args: str) -> list[str]:
+    return ["git", *GIT_SAFE, *args]
+
+
+def git_env() -> dict[str, str]:
+    env = child_env()
+    env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
+    return env
+
+
+def export_tree(src: Path, dest: Path) -> None:
+    """The checked-out commit of src and of its submodules, recursively, as
+    plain files in a fresh dest: git archive read by tarfile, which refuses
+    absolute paths, .. and links that leave dest."""
+    if dest.exists():
+        shutil.rmtree(dest)
+    dest.mkdir(parents=True)
+
+    def one(repo: Path, target: Path) -> None:
+        archive = run_group(host_git("-C", str(repo), "archive", "--format=tar", "HEAD"),
+                            capture_output=True, check=True, env=git_env(), timeout=1800)
+        with tarfile.open(fileobj=io.BytesIO(archive.stdout)) as handle:
+            if hasattr(tarfile, "data_filter"):
+                handle.extractall(target, filter="data")
+            else:
+                for member in handle.getmembers():
+                    name = Path(member.name)
+                    if name.is_absolute() or ".." in name.parts or member.islnk() or (
+                            member.issym() and (Path(member.linkname).is_absolute()
+                                                or ".." in Path(member.linkname).parts)):
+                        raise Unavailable(f"core: unsafe member {member.name} in {repo}")
+                handle.extractall(target)
+        listing = run_group(host_git("-C", str(repo), "ls-files", "--stage", "-z"),
+                            capture_output=True, check=True, env=git_env(), timeout=600)
+        for entry in listing.stdout.decode("utf-8", "replace").split("\0"):
+            if entry.startswith("160000 "):
+                path = entry.split("\t", 1)[1]
+                if (repo / path / ".git").exists():
+                    one(repo / path, target / path)
+
+    one(src, dest)
+
+
 @contextlib.contextmanager
 def working_directory(path: Path) -> Iterator[None]:
     """contextlib.chdir, which Python 3.9 lacks."""
@@ -641,36 +692,46 @@ class Cores:
         log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log"
         log.parent.mkdir(parents=True, exist_ok=True)
 
-        def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800) -> None:
+        def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800,
+                 env: dict[str, str] | None = None) -> None:
             with log.open("a", encoding="utf-8") as handle:
                 handle.write(f"$ {' '.join(argv)}\n")
                 handle.flush()
                 proc = self.c.host.run(argv, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT,
-                                       check=False, timeout=timeout, env=child_env())
+                                       check=False, timeout=timeout, env=env or git_env())
             if proc.returncode != 0:
                 tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:]
-                raise Unavailable(f"core: {argv[0]} {argv[1] if len(argv) > 1 else ''} exited "
+                raise Unavailable(f"core: {argv[0]} exited {proc.returncode}: "
-                                  f"{proc.returncode}: {' | '.join(tail)} (log {log})")
+                                  f"{' | '.join(tail)} (log {log})")
+        if core_probe.sandbox_argv([])[1] != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed:
+            raise Unavailable(f"refused: no sandbox to build {source} in "
+                              f"({core_probe.sandbox_argv([])[1]}), --allow-unsandboxed overrides")
         if not (src / ".git").is_dir():
             src.parent.mkdir(parents=True, exist_ok=True)
-            step(["git", "clone", "--quiet", source, str(src)])
+            step(host_git("clone", "--quiet", "--no-checkout", source, str(src)))
-        step(["git", "-C", str(src), "fetch", "--quiet", "origin", commit])
+        step(host_git("-C", str(src), "fetch", "--quiet", "origin", commit))
-        step(["git", "-C", str(src), "checkout", "--quiet", "--force", commit])
+        step(host_git("-C", str(src), "checkout", "--quiet", "--force", commit))
-        step(["git", "-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet"])
+        step(host_git("-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet"))
-        makefile = "Makefile.libretro" if (src / "Makefile.libretro").is_file() else "Makefile"
+        # The build runs on an exported tree. The clone and its .git are never
+        # bound into the sandbox, so a build cannot plant a hook or a config
+        # line that the next host git command would run.
+        tree = self.c.paths.home / "build-trees" / f"{slug(source)}-{commit[:12]}"
+        try:
+            export_tree(src, tree)
+        except (OSError, subprocess.CalledProcessError, tarfile.TarError) as exc:
+            raise Unavailable(f"core: exporting {commit[:12]} of {source} failed: {exc}") from exc
+        makefile = "Makefile.libretro" if (tree / "Makefile.libretro").is_file() else "Makefile"
         jobs = str(max(1, (os.cpu_count() or 2) // 2))
         command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"]
-        prefix, isolation = core_probe.sandbox_argv([src])
+        prefix, isolation = core_probe.sandbox_argv([tree])
-        if isolation != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed:
+        step(prefix + command, cwd=tree, timeout=BUILD_TIMEOUT, env=child_env())
-            raise Unavailable(f"refused: no sandbox to build {source} in ({isolation}), "
+        built = sorted(tree.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime)
-                              f"--allow-unsandboxed overrides")
-        step(prefix + command, cwd=src, timeout=BUILD_TIMEOUT)
-        built = sorted(src.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime)
         if not built:
             raise Unavailable(f"core: {' '.join(command)} produced no *_libretro.so (log {log})")
         out.mkdir(parents=True, exist_ok=True)
         shutil.copy2(built[-1], out / built[-1].name)
-        return out / built[-1].name, f"{' '.join(command)} (isolation: {isolation})"
+        shutil.rmtree(tree, ignore_errors=True)
+        return out / built[-1].name, f"{' '.join(command)} on git archive (isolation: {isolation})"
 
 
 # --------------------------------------------------------------------------

tests/test_campaign.py

@@ -468,6 +468,69 @@ class FailClosed(Workspace):
         self.assertFalse(any(call[0] == "make" for call in host.calls))
 
 
+@unittest.skipUnless(shutil.which("git") and shutil.which("make"), "no git or make")
+class BuildIsolation(unittest.TestCase):
+    """A build that writes into .git must not reach the next host git command."""
+
+    def setUp(self) -> None:
+        TMP.mkdir(parents=True, exist_ok=True)
+        self.tmp = Path(tempfile.mkdtemp(dir=TMP))
+        self.addCleanup(shutil.rmtree, self.tmp, True)
+        self.src = self.tmp / "src"
+        self.pwned = self.tmp / "PWNED"
+        hook = f"#!/bin/sh\ntouch {self.pwned}\n"
+        # A Makefile that tries to plant a hook and a config line in the clone.
+        makefile = (
+            "all:\n"
+            f"\t-mkdir -p {self.src}/.git/hooks ../.git/hooks\n"
+            f"\t-printf '{hook.replace(chr(10), '\\\\n')}' > {self.src}/.git/hooks/post-checkout\n"
+            f"\t-printf '[core]\\n\\tfsmonitor = touch {self.pwned}\\n' >> {self.src}/.git/config\n"
+            "\ttouch fake_libretro.so\n")
+        git = ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", str(self.src)]
+        subprocess.run(["git", "init", "-q", str(self.src)], check=True)
+        (self.src / "Makefile").write_text(makefile)
+        (self.src / "a.txt").write_text("one")
+        subprocess.run([*git, "add", "Makefile", "a.txt"], check=True)
+        subprocess.run([*git, "commit", "-q", "-m", "one"], check=True)
+        (self.src / "a.txt").write_text("two")
+        subprocess.run([*git, "commit", "-q", "-am", "two"], check=True)
+        self.config = (self.src / ".git" / "config").read_text()
+
+    def test_the_build_runs_on_an_export_without_git(self) -> None:
+        prefix, isolation = campaign.core_probe.sandbox_argv([self.tmp / "tree"])
+        if isolation != campaign.core_probe.SANDBOX:
+            self.skipTest("no working bwrap on this host")
+        tree = self.tmp / "tree"
+        campaign.export_tree(self.src, tree)
+        self.assertFalse((tree / ".git").exists())
+        self.assertEqual((tree / "a.txt").read_text(), "two")
+        proc = subprocess.run(prefix + ["make"], cwd=tree, capture_output=True, text=True,
+                              check=False, env=campaign.child_env())
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertTrue((tree / "fake_libretro.so").exists())
+        self.assertFalse((self.src / ".git" / "hooks" / "post-checkout").exists())
+        self.assertEqual((self.src / ".git" / "config").read_text(), self.config)
+        subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"),
+                       check=True, env=campaign.git_env())
+        self.assertFalse(self.pwned.exists())
+
+    def test_host_git_runs_no_planted_hook_or_fsmonitor(self) -> None:
+        hook = self.src / ".git" / "hooks" / "post-checkout"
+        hook.write_text(f"#!/bin/sh\ntouch {self.pwned}\n")
+        hook.chmod(0o755)
+        with (self.src / ".git" / "config").open("a") as handle:
+            handle.write(f"[core]\n\tfsmonitor = touch {self.pwned}\n")
+        subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"),
+                       check=True, env=campaign.git_env())
+        subprocess.run(campaign.host_git("-C", str(self.src), "status", "--short"),
+                       check=True, env=campaign.git_env(), capture_output=True)
+        self.assertFalse(self.pwned.exists())
+        # The same checkout with plain git does run the hook: the test plants a live one.
+        subprocess.run(["git", "-C", str(self.src), "checkout", "-q", "HEAD@{1}"], check=True,
+                       capture_output=True)
+        self.assertTrue(self.pwned.exists())
+
+
 class ProbeRuns(Workspace):
     def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess:
         return subprocess.CompletedProcess([], code, stdout, stderr)

wiki/tools.md

@@ -955,7 +955,7 @@ python scripts/campaign.py install-timer     # the autorun units, the campaign's
 
 | Step | What happens |
 |------|--------------|
-| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. git clones with the allowlisted environment, and `make` runs in the same bubblewrap sandbox as a core, its source tree the one writable folder. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` |
+| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. git clones with the allowlisted environment, no user or system config, and `core.hooksPath=/dev/null`, `core.fsmonitor=false` and the `file` and `ext` transports refused. The commit and its submodules are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path: the clone and its `.git` are never inside, so a build cannot plant a hook or a config line for the next host git command. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` |
 | Content | Each system's games come from `tests/campaign/content.yml`, by catalog name and with the reason each was chosen. They are read from `.cache/test-content/<romfetch system>/<name>/`, each file checked against its `identity.json`. Missing content is fetched with `romfetch get <system> <name> --json` when romfetch is on `PATH`, in `RETROBIOS_ROMFETCH` or given with `--romfetch`. Otherwise the checks are `pending: content`, with romfetch's exit code and error code when it failed. Content never enters git, and a record keeps only its identity |
 | Checks | `files` stages every declared file the collection holds and reads which ones the core opens (strace). `boot` runs the game with no file and with the declared one (`boot_file:`, else the entry of the game's region). An entry the `files` run left unread is staged `alone`, then where the core looked for it (`path`) and under the names it looked for (`alias`). `variants` runs each `.variants/` copy of a file the core opened. `bench` times the game, warm-up excluded, 5 repetitions |
 | Rules | `campaign_rules.py`, a fixed table of pure functions (below). A field the table settles is written into the profile, with `evidence:` naming the record ids. Anything else is a line of `measurements/findings.jsonl` |