diff --git a/scripts/campaign.py b/scripts/campaign.py index 081dc1339..f1ac972cb 100644 --- a/scripts/campaign.py +++ b/scripts/campaign.py @@ -44,6 +44,7 @@ import argparse import contextlib import datetime as dt import hashlib +import io import json import os import re @@ -51,6 +52,7 @@ import shutil import signal import subprocess import sys +import tarfile import time import urllib.error import urllib.request @@ -282,6 +284,55 @@ def child_env(romfetch: bool = False) -> dict[str, str]: return autorun.safe_env(names) +# Host-side git on a clone of a third-party project: no hook, no fsmonitor, +# no local-file transport, no user or system config. +GIT_SAFE = ("-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", + "-c", "protocol.file.allow=never", "-c", "protocol.ext.allow=never") + + +def host_git(*args: str) -> list[str]: + return ["git", *GIT_SAFE, *args] + + +def git_env() -> dict[str, str]: + env = child_env() + env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0") + return env + + +def export_tree(src: Path, dest: Path) -> None: + """The checked-out commit of src and of its submodules, recursively, as + plain files in a fresh dest: git archive read by tarfile, which refuses + absolute paths, .. and links that leave dest.""" + if dest.exists(): + shutil.rmtree(dest) + dest.mkdir(parents=True) + + def one(repo: Path, target: Path) -> None: + archive = run_group(host_git("-C", str(repo), "archive", "--format=tar", "HEAD"), + capture_output=True, check=True, env=git_env(), timeout=1800) + with tarfile.open(fileobj=io.BytesIO(archive.stdout)) as handle: + if hasattr(tarfile, "data_filter"): + handle.extractall(target, filter="data") + else: + for member in handle.getmembers(): + name = Path(member.name) + if name.is_absolute() or ".." in name.parts or member.islnk() or ( + member.issym() and (Path(member.linkname).is_absolute() + or ".." in Path(member.linkname).parts)): + raise Unavailable(f"core: unsafe member {member.name} in {repo}") + handle.extractall(target) + listing = run_group(host_git("-C", str(repo), "ls-files", "--stage", "-z"), + capture_output=True, check=True, env=git_env(), timeout=600) + for entry in listing.stdout.decode("utf-8", "replace").split("\0"): + if entry.startswith("160000 "): + path = entry.split("\t", 1)[1] + if (repo / path / ".git").exists(): + one(repo / path, target / path) + + one(src, dest) + + @contextlib.contextmanager def working_directory(path: Path) -> Iterator[None]: """contextlib.chdir, which Python 3.9 lacks.""" @@ -641,36 +692,46 @@ class Cores: log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log" log.parent.mkdir(parents=True, exist_ok=True) - def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800) -> None: + def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800, + env: dict[str, str] | None = None) -> None: with log.open("a", encoding="utf-8") as handle: handle.write(f"$ {' '.join(argv)}\n") handle.flush() proc = self.c.host.run(argv, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT, - check=False, timeout=timeout, env=child_env()) + check=False, timeout=timeout, env=env or git_env()) if proc.returncode != 0: tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:] - raise Unavailable(f"core: {argv[0]} {argv[1] if len(argv) > 1 else ''} exited " - f"{proc.returncode}: {' | '.join(tail)} (log {log})") + raise Unavailable(f"core: {argv[0]} exited {proc.returncode}: " + f"{' | '.join(tail)} (log {log})") + if core_probe.sandbox_argv([])[1] != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed: + raise Unavailable(f"refused: no sandbox to build {source} in " + f"({core_probe.sandbox_argv([])[1]}), --allow-unsandboxed overrides") if not (src / ".git").is_dir(): src.parent.mkdir(parents=True, exist_ok=True) - step(["git", "clone", "--quiet", source, str(src)]) - step(["git", "-C", str(src), "fetch", "--quiet", "origin", commit]) - step(["git", "-C", str(src), "checkout", "--quiet", "--force", commit]) - step(["git", "-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet"]) - makefile = "Makefile.libretro" if (src / "Makefile.libretro").is_file() else "Makefile" + step(host_git("clone", "--quiet", "--no-checkout", source, str(src))) + step(host_git("-C", str(src), "fetch", "--quiet", "origin", commit)) + step(host_git("-C", str(src), "checkout", "--quiet", "--force", commit)) + step(host_git("-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet")) + # The build runs on an exported tree. The clone and its .git are never + # bound into the sandbox, so a build cannot plant a hook or a config + # line that the next host git command would run. + tree = self.c.paths.home / "build-trees" / f"{slug(source)}-{commit[:12]}" + try: + export_tree(src, tree) + except (OSError, subprocess.CalledProcessError, tarfile.TarError) as exc: + raise Unavailable(f"core: exporting {commit[:12]} of {source} failed: {exc}") from exc + makefile = "Makefile.libretro" if (tree / "Makefile.libretro").is_file() else "Makefile" jobs = str(max(1, (os.cpu_count() or 2) // 2)) command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"] - prefix, isolation = core_probe.sandbox_argv([src]) - if isolation != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed: - raise Unavailable(f"refused: no sandbox to build {source} in ({isolation}), " - f"--allow-unsandboxed overrides") - step(prefix + command, cwd=src, timeout=BUILD_TIMEOUT) - built = sorted(src.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime) + prefix, isolation = core_probe.sandbox_argv([tree]) + step(prefix + command, cwd=tree, timeout=BUILD_TIMEOUT, env=child_env()) + built = sorted(tree.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime) if not built: raise Unavailable(f"core: {' '.join(command)} produced no *_libretro.so (log {log})") out.mkdir(parents=True, exist_ok=True) shutil.copy2(built[-1], out / built[-1].name) - return out / built[-1].name, f"{' '.join(command)} (isolation: {isolation})" + shutil.rmtree(tree, ignore_errors=True) + return out / built[-1].name, f"{' '.join(command)} on git archive (isolation: {isolation})" # -------------------------------------------------------------------------- diff --git a/tests/test_campaign.py b/tests/test_campaign.py index bb31cddc3..6354bfeb5 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -468,6 +468,69 @@ class FailClosed(Workspace): self.assertFalse(any(call[0] == "make" for call in host.calls)) +@unittest.skipUnless(shutil.which("git") and shutil.which("make"), "no git or make") +class BuildIsolation(unittest.TestCase): + """A build that writes into .git must not reach the next host git command.""" + + def setUp(self) -> None: + TMP.mkdir(parents=True, exist_ok=True) + self.tmp = Path(tempfile.mkdtemp(dir=TMP)) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.src = self.tmp / "src" + self.pwned = self.tmp / "PWNED" + hook = f"#!/bin/sh\ntouch {self.pwned}\n" + # A Makefile that tries to plant a hook and a config line in the clone. + makefile = ( + "all:\n" + f"\t-mkdir -p {self.src}/.git/hooks ../.git/hooks\n" + f"\t-printf '{hook.replace(chr(10), '\\\\n')}' > {self.src}/.git/hooks/post-checkout\n" + f"\t-printf '[core]\\n\\tfsmonitor = touch {self.pwned}\\n' >> {self.src}/.git/config\n" + "\ttouch fake_libretro.so\n") + git = ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", str(self.src)] + subprocess.run(["git", "init", "-q", str(self.src)], check=True) + (self.src / "Makefile").write_text(makefile) + (self.src / "a.txt").write_text("one") + subprocess.run([*git, "add", "Makefile", "a.txt"], check=True) + subprocess.run([*git, "commit", "-q", "-m", "one"], check=True) + (self.src / "a.txt").write_text("two") + subprocess.run([*git, "commit", "-q", "-am", "two"], check=True) + self.config = (self.src / ".git" / "config").read_text() + + def test_the_build_runs_on_an_export_without_git(self) -> None: + prefix, isolation = campaign.core_probe.sandbox_argv([self.tmp / "tree"]) + if isolation != campaign.core_probe.SANDBOX: + self.skipTest("no working bwrap on this host") + tree = self.tmp / "tree" + campaign.export_tree(self.src, tree) + self.assertFalse((tree / ".git").exists()) + self.assertEqual((tree / "a.txt").read_text(), "two") + proc = subprocess.run(prefix + ["make"], cwd=tree, capture_output=True, text=True, + check=False, env=campaign.child_env()) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue((tree / "fake_libretro.so").exists()) + self.assertFalse((self.src / ".git" / "hooks" / "post-checkout").exists()) + self.assertEqual((self.src / ".git" / "config").read_text(), self.config) + subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"), + check=True, env=campaign.git_env()) + self.assertFalse(self.pwned.exists()) + + def test_host_git_runs_no_planted_hook_or_fsmonitor(self) -> None: + hook = self.src / ".git" / "hooks" / "post-checkout" + hook.write_text(f"#!/bin/sh\ntouch {self.pwned}\n") + hook.chmod(0o755) + with (self.src / ".git" / "config").open("a") as handle: + handle.write(f"[core]\n\tfsmonitor = touch {self.pwned}\n") + subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"), + check=True, env=campaign.git_env()) + subprocess.run(campaign.host_git("-C", str(self.src), "status", "--short"), + check=True, env=campaign.git_env(), capture_output=True) + self.assertFalse(self.pwned.exists()) + # The same checkout with plain git does run the hook: the test plants a live one. + subprocess.run(["git", "-C", str(self.src), "checkout", "-q", "HEAD@{1}"], check=True, + capture_output=True) + self.assertTrue(self.pwned.exists()) + + class ProbeRuns(Workspace): def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess: return subprocess.CompletedProcess([], code, stdout, stderr) diff --git a/wiki/tools.md b/wiki/tools.md index 6309a011a..ca3a13114 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -955,7 +955,7 @@ python scripts/campaign.py install-timer # the autorun units, the campaign's | Step | What happens | |------|--------------| -| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. git clones with the allowlisted environment, and `make` runs in the same bubblewrap sandbox as a core, its source tree the one writable folder. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` | +| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. git clones with the allowlisted environment, no user or system config, and `core.hooksPath=/dev/null`, `core.fsmonitor=false` and the `file` and `ext` transports refused. The commit and its submodules are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path: the clone and its `.git` are never inside, so a build cannot plant a hook or a config line for the next host git command. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` | | Content | Each system's games come from `tests/campaign/content.yml`, by catalog name and with the reason each was chosen. They are read from `.cache/test-content///`, each file checked against its `identity.json`. Missing content is fetched with `romfetch get --json` when romfetch is on `PATH`, in `RETROBIOS_ROMFETCH` or given with `--romfetch`. Otherwise the checks are `pending: content`, with romfetch's exit code and error code when it failed. Content never enters git, and a record keeps only its identity | | Checks | `files` stages every declared file the collection holds and reads which ones the core opens (strace). `boot` runs the game with no file and with the declared one (`boot_file:`, else the entry of the game's region). An entry the `files` run left unread is staged `alone`, then where the core looked for it (`path`) and under the names it looked for (`alias`). `variants` runs each `.variants/` copy of a file the core opened. `bench` times the game, warm-up excluded, 5 repetitions | | Rules | `campaign_rules.py`, a fixed table of pure functions (below). A field the table settles is written into the profile, with `evidence:` naming the record ids. Anything else is a line of `measurements/findings.jsonl` |