diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json index 00594716d..4fa4d1737 100644 --- a/schemas/core-probe-record.schema.json +++ b/schemas/core-probe-record.schema.json @@ -686,6 +686,10 @@ "tracing": { "type": "string" }, + "isolation": { + "type": "string", + "description": "How the core ran: `bwrap` (read-only tree, no network, the user's credentials masked, an empty environment beside the --env values) or `environment only: ...` with the reason bubblewrap was not used." + }, "input_queries": { "type": "integer", "minimum": 0 diff --git a/scripts/campaign.py b/scripts/campaign.py index ebdff09a3..908065527 100644 --- a/scripts/campaign.py +++ b/scripts/campaign.py @@ -48,6 +48,7 @@ import json import os import re import shutil +import signal import subprocess import sys import time @@ -65,6 +66,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import autorun import campaign_profiles import campaign_rules as rules +import core_probe import measurement_records as mr ROOT = Path(__file__).resolve().parent.parent @@ -177,6 +179,27 @@ def is_bios(entry: dict) -> bool: # Host # -------------------------------------------------------------------------- +def run_group(argv: Sequence[str], *, timeout: float | None = None, check: bool = False, + capture_output: bool = False, **kwargs: object) -> subprocess.CompletedProcess: + """subprocess.run in a process group of its own, the whole group killed + on a timeout or an interruption: a hung core never outlives its run.""" + if capture_output: + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + with subprocess.Popen(list(argv), start_new_session=True, **kwargs) as proc: + try: + out, err = proc.communicate(timeout=timeout) + except BaseException: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.communicate() + raise + result = subprocess.CompletedProcess(list(argv), proc.returncode, out, err) + if check: + result.check_returncode() + return result + + @dataclass class Host: """What the campaign reads from the machine and the network, replaced in tests.""" @@ -187,7 +210,7 @@ class Host: cpu_count: int = os.cpu_count() or 1 free_gb: Callable[[Path], float] = autorun.free_gb poll: float = 60.0 - run: Callable[..., subprocess.CompletedProcess] = subprocess.run + run: Callable[..., subprocess.CompletedProcess] = run_group urlopen: Callable[..., object] = urllib.request.urlopen @property @@ -242,6 +265,30 @@ class Paths: content=main / ".cache" / "test-content", db=ROOT / "database.json") +ROMFETCH_PASSED = ("XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME") + + +def child_env(romfetch: bool = False) -> dict[str, str]: + """What a program the campaign starts inherits: autorun's allowlist, + never a token or a password. romfetch also gets its own settings + (ROMFETCH_*), its account staying in its config file.""" + names: list[str] = [] + if romfetch: + names = [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED) + return autorun.safe_env(names) + + +@contextlib.contextmanager +def working_directory(path: Path) -> Iterator[None]: + """contextlib.chdir, which Python 3.9 lacks.""" + before = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(before) + + @contextlib.contextmanager def host_lock(lock_path: Path) -> Iterator[None]: """The automation lock, unless the autorun job running us holds it.""" @@ -405,7 +452,7 @@ class ContentStore: "--content-dir", str(self.c.paths.content)] for attempt, delay in enumerate((*self.c.settings.delays, None)): proc = self.c.host.run(argv, capture_output=True, text=True, check=False, - timeout=6 * 3600) + timeout=6 * 3600, env=child_env(romfetch=True)) if proc.returncode == 0: self.c.log(f"romfetch {content.name}: {proc.stdout.strip()[:300]}") return @@ -569,17 +616,23 @@ class Cores: if not self.c.settings.network: raise Unavailable("core: not on the buildbot, offline, nothing built yet") try: - built = [self.compile(source, commit, out)] + so, command = self.compile(source, commit, out) except Unavailable as exc: self.c.state["builds"][emulator] = {"commit": commit, "error": str(exc), "at": now_iso()} raise + built = [so] + record["command"] = command command = record.get("command") or "make" self.c.state["builds"][emulator] = {"commit": commit, "command": command, "so": built[0].name, "at": now_iso()} return Core(built[0].name.removesuffix("_libretro.so"), built[0], file_sha256(built[0]), repo=source, commit=commit, build_command=command) - def compile(self, source: str, commit: str, out: Path) -> Path: + def compile(self, source: str, commit: str, out: Path) -> tuple[Path, str]: + """Clone with git (trusted, networked), then run the project's make + under bubblewrap: no network, a read-only tree with the source + writable, the user's credentials masked. Returns the .so and the + command, with how it was isolated.""" src = self.c.paths.home / "src" / slug(source) log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log" log.parent.mkdir(parents=True, exist_ok=True) @@ -589,7 +642,7 @@ class Cores: handle.write(f"$ {' '.join(argv)}\n") handle.flush() proc = self.c.host.run(argv, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT, - check=False, timeout=timeout) + check=False, timeout=timeout, env=child_env()) if proc.returncode != 0: tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:] raise Unavailable(f"core: {argv[0]} {argv[1] if len(argv) > 1 else ''} exited " @@ -603,13 +656,14 @@ class Cores: makefile = "Makefile.libretro" if (src / "Makefile.libretro").is_file() else "Makefile" jobs = str(max(1, (os.cpu_count() or 2) // 2)) command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"] - step(command, cwd=src, timeout=BUILD_TIMEOUT) + prefix, isolation = core_probe.sandbox_argv([src], private_tmp=True) + step(prefix + command, cwd=src, timeout=BUILD_TIMEOUT) built = sorted(src.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime) if not built: raise Unavailable(f"core: {' '.join(command)} produced no *_libretro.so (log {log})") out.mkdir(parents=True, exist_ok=True) shutil.copy2(built[-1], out / built[-1].name) - return out / built[-1].name + return out / built[-1].name, f"{' '.join(command)} (isolation: {isolation})" # -------------------------------------------------------------------------- @@ -699,7 +753,7 @@ class Campaign: hint = entry.get("path") if isinstance(entry.get("path"), str) else "" # The database names files relative to the checkout it describes. - with contextlib.chdir(self.paths.root): + with working_directory(self.paths.root): path, status = resolve_local_file(entry, self.db, dest_hint=hint or "") if not path or status in ("not_found", "hash_mismatch"): return None @@ -754,7 +808,7 @@ class Campaign: self.guard() try: proc = self.host.run(command, capture_output=True, text=True, check=False, - timeout=budget, cwd=self.paths.root) + timeout=budget, cwd=self.paths.root, env=child_env()) except subprocess.TimeoutExpired: with log.open("a", encoding="utf-8") as handle: handle.write(f"$ {' '.join(command)}\nkilled after {budget} s\n") @@ -1492,6 +1546,11 @@ def main(argv: Sequence[str] | None = None) -> int: campaign.run(only, args.system, dry_run=True) return 0 campaign.dirty_at_start = set(autorun.changed_files(paths.root)) if args.commit else set() + + def stop(signum: int, frame: object) -> None: + # A timer's TimeoutStartSec ends the pass like Ctrl-C: state saved, pass line written. + raise KeyboardInterrupt + signal.signal(signal.SIGTERM, stop) try: with host_lock(campaign.lock_path): current = campaign.run(only, args.system, commit_every=args.commit_every, diff --git a/scripts/core_probe.py b/scripts/core_probe.py index 846efd602..140988979 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -18,6 +18,7 @@ reproduce it (core build, frontend commit, host, inputs by hash). from __future__ import annotations import argparse +import contextlib import ctypes import datetime as dt import hashlib @@ -26,6 +27,7 @@ import os import platform import re import shutil +import signal import statistics import struct import subprocess @@ -470,6 +472,9 @@ class Frontend: def run_worker(spec: dict) -> dict: """Load the core in this process and run it. Returns the measured result.""" + # Nothing of the host's environment reaches the core: only what --env gave. + os.environ.clear() + os.environ.update(spec["env"]) core = ctypes.CDLL(spec["core"]) front = Frontend(spec) for name, fn in zip(("retro_set_environment", "retro_set_video_refresh", "retro_set_audio_sample", @@ -646,6 +651,59 @@ def host_paths_hidden(text: str, places: Sequence[tuple[str, str]]) -> str: return text +_BWRAP_WORKS: dict[str, bool] = {} + + +def sandbox_argv(writable: Sequence[Path], private_tmp: bool = False) -> tuple[list[str], str]: + """The bubblewrap prefix a third-party program runs under, and its name. + + The whole tree is read-only, the network and the other namespaces are + unshared, the user's credentials (~/.config, ~/.ssh, ~/.gnupg, + ~/.git-credentials, ~/.netrc) are masked, and only `writable` can be + written. Without a working bwrap the program still runs, with an empty + environment, and the returned name says so. + """ + bwrap = shutil.which("bwrap") + if not bwrap: + return [], "environment only: bwrap not found" + base = [bwrap, "--ro-bind", "/", "/", "--dev", "/dev", "--tmpfs", "/dev/shm", "--proc", "/proc", + "--unshare-all", "--die-with-parent", "--new-session"] + if bwrap not in _BWRAP_WORKS: + probe_run = subprocess.run(base + ["true"], capture_output=True, check=False, env={}) + _BWRAP_WORKS[bwrap] = probe_run.returncode == 0 + if not _BWRAP_WORKS[bwrap]: + return [], "environment only: bwrap cannot create a sandbox here" + home = Path.home() + argv = list(base) + (["--tmpfs", "/tmp"] if private_tmp else []) + for hidden in (home / ".config", home / ".ssh", home / ".gnupg"): + if hidden.is_dir(): + argv += ["--tmpfs", str(hidden)] + for secret in (home / ".git-credentials", home / ".netrc"): + if secret.exists(): + argv += ["--ro-bind", "/dev/null", str(secret)] + for path in writable: + resolved = Path(path).resolve() + resolved.mkdir(parents=True, exist_ok=True) + argv += ["--bind", str(resolved), str(resolved)] + return argv, "bwrap" + + +def run_group(command: list[str], timeout: float, env: dict[str, str]) -> subprocess.CompletedProcess: + """Run the worker in a process group of its own and kill the whole group + when it outlives the timeout. Killing strace alone left a hung core + spinning, detached from anything that would ever stop it.""" + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=env, start_new_session=True) as proc: + try: + out, err = proc.communicate(timeout=timeout) + except BaseException: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.communicate() + raise + return subprocess.CompletedProcess(command, proc.returncode, out, err) + + def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None, png_dir: Path | None) -> dict: save_dir = scratch / "save" @@ -664,6 +722,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou "joypad_ports": args.joypad_ports, "png_dir": str(png_dir) if png_dir else None, "state_out": str(state_out) if state_out else None, + "env": dict(args.env), } spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt" spec_path.write_text(json.dumps(spec), encoding="utf-8") @@ -671,9 +730,10 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou strace = shutil.which("strace") if not args.no_trace else None if strace: command = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace_path)] + command - env = {**os.environ, **dict(args.env)} - proc = subprocess.run(command, check=False, capture_output=True, text=True, - timeout=args.timeout, env=env) + writable = [scratch, system_dir] + [p for p in (png_dir, state_out.parent if state_out else None) if p] + prefix, isolation = sandbox_argv(writable) + # The core sees the variables it was given and nothing of the host's. + proc = run_group(prefix + command, args.timeout, dict(args.env)) if proc.returncode != 0 or not out_path.exists(): raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}") result = json.loads(out_path.read_text(encoding="utf-8")) @@ -697,6 +757,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou if result.get("wall_s") and result.get("frames"): result["ms_per_frame"] = result["wall_s"] * 1000 / result["frames"] result["fps"] = result["frames"] / result["wall_s"] + result["isolation"] = isolation return result @@ -824,6 +885,20 @@ def build_parser() -> argparse.ArgumentParser: return parser +def not_a_checkout_root(directory: Path) -> str | None: + """Why --record DIR is refused, or None. DIR is the checkout, never its + measurements/ folder: that wrote measurements/measurements/.jsonl.""" + form = "--record names the root of a git checkout (the folder holding measurements/), e.g. --record ." + if not (directory / ".git").exists(): + return f"--record {directory}: not a git checkout root. {form}" + proc = subprocess.run(["git", "-C", str(directory), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=False) + top = proc.stdout.strip() + if proc.returncode != 0 or not top or Path(top).resolve() != directory.resolve(): + return f"--record {directory}: not a git checkout root (its root is {top or 'unknown'}). {form}" + return None + + def validate_args(args: argparse.Namespace) -> list[str]: problems: list[str] = [] if not Path(args.core).is_file(): @@ -838,6 +913,9 @@ def validate_args(args: argparse.Namespace) -> list[str]: if args.png_dir and not args.frame_hash_every: problems.append("--png-dir needs --frame-hash-every") if args.record: + wrong = not_a_checkout_root(Path(args.record)) + if wrong: + problems.append(wrong) if not args.emulator and Path(args.core).is_file(): try: default_emulator(Path(args.core), None) diff --git a/tests/test_campaign.py b/tests/test_campaign.py index bb721b5a6..359931812 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -20,6 +20,7 @@ import subprocess import sys import tempfile import unittest +import unittest.mock from pathlib import Path from types import MappingProxyType @@ -95,12 +96,19 @@ bool retro_load_game(const void *g) { const char *dir = NULL; FILE *f; (void)g; env(9, &dir); f = try_open(dir, "bios.bin"); if (!f) f = try_open(dir, "BIOS.BIN"); - if (f) { fclose(f); booted = 1; } else booted = getenv("FAKE_HLE") != NULL; +#ifdef FAKE_HLE + if (f) fclose(f); + booted = 1; +#else + if (f) { fclose(f); booted = 1; } else booted = 0; +#endif return true; } void retro_unload_game(void) {} void retro_run(void) { int16_t s[4] = {1, 2, 3, 4}; - if (getenv("FAKE_HANG")) for (;;) {} +#ifdef FAKE_HANG + for (;;) {} +#endif for (int i = 0; i < 8; i++) pixels[i] = booted ? (uint16_t)(frame / 2 + i) : 0; frame++; video(pixels, 4, 2, 8); batch(s, 2); } @@ -284,6 +292,7 @@ class Workspace(unittest.TestCase): self.tmp = Path(tempfile.mkdtemp(dir=TMP)) self.root = self.tmp / "repo" (self.root / "emulators").mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(self.root)], check=True) (self.root / "emulators" / "fake.yml").write_text(PROFILE, encoding="utf-8") bios = self.root / "bios" / "Fake" / "bios.bin" bios.parent.mkdir(parents=True) @@ -392,6 +401,34 @@ class Guards(Workspace): del os.environ[campaign.LOCK_HELD_ENV] +class ChildEnvironment(unittest.TestCase): + def test_probes_builds_and_romfetch_get_no_secret(self) -> None: + host = {"GITHUB_TOKEN": "t", "GH_TOKEN": "t", "ROMSTATION_USER": "u", "ROMSTATION_PASS": "p", + "ROMFETCH_CONTENT_DIR": "/c", "SSH_AUTH_SOCK": "/s", "PATH": "/usr/bin"} + with unittest.mock.patch.dict(os.environ, host): + probe, fetch = campaign.child_env(), campaign.child_env(romfetch=True) + for env in (probe, fetch): + self.assertFalse({"GITHUB_TOKEN", "GH_TOKEN", "ROMSTATION_USER", "ROMSTATION_PASS", + "SSH_AUTH_SOCK"} & set(env)) + self.assertEqual(env["PATH"], "/usr/bin") + self.assertNotIn("ROMFETCH_CONTENT_DIR", probe) + self.assertEqual(fetch["ROMFETCH_CONTENT_DIR"], "/c") + + def test_the_probe_is_started_with_that_environment(self) -> None: + seen: list[dict] = [] + host = FakeHost() + host.run = lambda argv, **kw: (seen.append(kw.get("env")), + subprocess.CompletedProcess([], 0, '{"id": "x"}', ""))[1] + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + paths = campaign.Paths(root=Path(tmp), home=Path(tmp) / "h", content=Path(tmp) / "c", + db=Path(tmp) / "db.json", content_file=campaign.CONTENT_FILE) + c = campaign.Campaign(paths, campaign.Settings(echo=False), host, lock_path=Path(tmp) / "l") + c.deadline = 10_000 + with unittest.mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}): + c.probe(["run"], "x") + self.assertNotIn("GITHUB_TOKEN", seen[0]) + + class ProbeRuns(Workspace): def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess: return subprocess.CompletedProcess([], code, stdout, stderr) @@ -649,10 +686,33 @@ class SitePage(unittest.TestCase): page = "\n".join(sitemeasurements.campaign_section(summary)) self.assertIn("12 tests done, 1 failed, 3 waiting", page) self.assertIn("| standalone | 3 |", page) - self.assertIn("the profile says `true`, the run shows `false`", page) + self.assertIn("the profile says true, the run shows false", page) self.assertIn("[2026-10-27-aaaaaaaaaaaa](https://x/l#L1)", page) self.assertEqual(sitemeasurements.campaign_section({"last_run": None}), []) + def test_observed_values_cannot_leave_their_place(self) -> None: + # A value comes from a core's output or a romfetch error: it may hold + # backticks, MDX braces and HTML. + hostile = "``}{import x}`` | [a](javascript:x) *b*" + summary = { + "last_run": {"ended_at": "2026-10-27T10:00:00+00:00", "result": "paused", "detail": hostile, + "counts": {"done": 0, "failed": 1, "pending": 1}, + "pending_reasons": {hostile: 1}, + "failed": [{"emulator": hostile, "check": hostile, "reason": hostile}]}, + "passes": 1, "applied": 0, + "open": [{"emulator": "fake", "file": hostile, "field": hostile, "status": "open", + "profile_says": {"k": hostile}, "run_shows": [hostile], "statement": hostile, + "evidence": ["2026-10-27-aaaaaaaaaaaa"], "links": [None]}], + } + page = "\n".join(sitemeasurements.campaign_section(summary)) + for raw in (" None: shutil.rmtree(cls.build, ignore_errors=True) - def passing(self, env: dict[str, str] | None = None, core: Path | None = None, + def passing(self, kind: str = "plain", core: Path | None = None, **settings: object) -> campaign.Campaign: self.put_game() # The core_probe threshold follows the host: a test never waits on the real load. host = campaign.Host(loadavg=lambda: (0.0, 0.0, 0.0), free_gb=lambda path: 500.0, cpu_count=1_000_000) - extra = env or {} - - def run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess: - check = bool(kwargs.pop("check", False)) - return subprocess.run(argv, check=check, env={**os.environ, **extra}, **kwargs) - host.run = run values = {"bench_warmup": 2, "bench_frames": 4, "check_repeat": 2, "probe_timeout": 30, "delays": (1,), "echo": False, **settings} c = campaign.Campaign(self.paths, campaign.Settings(**values), host, lock_path=self.tmp / "auto.lock") - so = core or self.core + so = core or self.cores[kind] c.unit_cores = lambda *a, **k: [campaign.Core("fake", so, campaign.file_sha256(so))] return c def test_a_stand_in_is_proven_and_written(self) -> None: - c = self.passing({"FAKE_HLE": "1"}) + c = self.passing("hle") current = c.run(set(), None) self.assertEqual(current.result, "complete") records = [s.record for s in mr.read_records(self.root)] @@ -712,15 +770,15 @@ class FakeCorePasses(Workspace): self.assertTrue(self.game.is_dir()) # A second pass on the same core and profile runs nothing. - again = self.passing({"FAKE_HLE": "1"}) + again = self.passing("hle") self.assertEqual(again.run(set(), None).records, []) # A new build of the core runs everything again. rebuilt = self.build / "fake2_libretro.so" - shutil.copy2(self.core, rebuilt) + shutil.copy2(self.cores["hle"], rebuilt) with rebuilt.open("ab") as handle: handle.write(b"\0") - third = self.passing({"FAKE_HLE": "1"}, core=rebuilt) + third = self.passing("hle", core=rebuilt) # BIOS.BIN is declared now, so no alias run follows the boot test. self.assertEqual(len(third.run(set(), None).records), 3) @@ -735,8 +793,11 @@ class FakeCorePasses(Workspace): self.assertEqual(alias[0]["status"], "applied") def test_a_hung_core_is_killed_and_the_pass_moves_on(self) -> None: - c = self.passing({"FAKE_HANG": "1"}, probe_timeout=2) + c = self.passing("hang", probe_timeout=2) c.run(set(), None) + # The whole process group went with the run: no worker spins on. + left = subprocess.run(["pgrep", "-f", str(self.cores["hang"])], capture_output=True, check=False) + self.assertEqual(left.stdout, b"") checks = c.state["checks"]["fake"] self.assertTrue(checks) self.assertEqual({entry["status"] for entry in checks.values()}, {"failed"}) diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index 49399ad06..493db3cee 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -11,6 +11,7 @@ import argparse import contextlib import io import json +import os import shutil import struct import subprocess @@ -99,6 +100,13 @@ def mednafen_state(sections: dict[str, dict[str, bytes]]) -> bytes: return b"MDFNSVST".ljust(32, b"\0") + body +def checkout(path: Path) -> Path: + """A folder made the root of a git checkout, as --record requires.""" + path.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init", "-q", str(path)], check=True) + return path + + class ParsingTests(unittest.TestCase): def test_assignment(self) -> None: self.assertEqual(core_probe.parse_assignment("a=b=c", "--option"), ("a", "b=c")) @@ -244,7 +252,7 @@ class FakeCoreTests(unittest.TestCase): return code, json.loads(out.getvalue()) if code == 0 else None def test_run_record(self) -> None: - record_dir = self.tmp / "records" + record_dir = checkout(self.tmp / "records") code, record = self.probe(["check", "--core", str(self.core), "--frames", "3", "--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={self.cards[1]}", "--vary", f"card.bin={self.cards[2]}", "--vary", "card.bin=", @@ -268,7 +276,8 @@ class FakeCoreTests(unittest.TestCase): self.assertEqual(first["frames"], 3) self.assertEqual(record["emulator"], "fake") self.assertEqual([p.relative_to(record_dir).as_posix() - for p in record_dir.rglob("*") if p.is_file()], ["measurements/fake.jsonl"]) + for p in record_dir.rglob("*") if p.is_file() and ".git" not in p.parts], + ["measurements/fake.jsonl"]) line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text()) self.assertEqual(measurement_records.check_line(line), []) # The line is the record the run printed, id included. @@ -281,7 +290,7 @@ class FakeCoreTests(unittest.TestCase): self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), []) def test_second_run_is_appended_beside_the_first(self) -> None: - record_dir = self.tmp / "appended" + record_dir = checkout(self.tmp / "appended") argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000", "--record", str(record_dir), "--subject", "smoke"] self.assertEqual(self.probe(argv)[0], 0) @@ -313,10 +322,80 @@ class FakeCoreTests(unittest.TestCase): self.assertEqual(record["flags"], []) def test_busy_machine_refused(self) -> None: + refused = checkout(self.tmp / "refused") code, _ = self.probe(["run", "--core", str(self.core), "--frames", "1", "--record", - str(self.tmp / "refused"), "--max-load", "-1"]) + str(refused), "--max-load", "-1"]) self.assertEqual(code, 3) - self.assertFalse((self.tmp / "refused").exists()) + self.assertFalse((refused / "measurements").exists()) + + def test_record_names_a_checkout_root(self) -> None: + # --record measurements wrote measurements/measurements/.jsonl. + root = checkout(self.tmp / "rooted") + (root / "measurements").mkdir() + for wrong in (root / "measurements", self.tmp / "plain"): + wrong.mkdir(exist_ok=True) + err = io.StringIO() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + code = core_probe.main(["run", "--core", str(self.core), "--frames", "1", "--no-trace", + "--record", str(wrong), "--work-dir", str(self.tmp / "work")]) + self.assertEqual(code, 2) + self.assertIn("not a git checkout root", err.getvalue()) + self.assertIn("--record .", err.getvalue()) + self.assertFalse((wrong / "measurements" / "fake.jsonl").exists()) + self.assertIsNone(core_probe.not_a_checkout_root(root)) + self.assertIsNone(core_probe.not_a_checkout_root(REPO_ROOT)) + + def test_the_core_sees_only_the_env_it_was_given(self) -> None: + os.environ["CORE_PROBE_TEST_TAG"] = "from-the-host" + os.environ["GITHUB_TOKEN"] = "secret" + try: + code, record = self.probe(["run", "--core", str(self.core), "--frames", "1", + "--no-trace"]) + finally: + del os.environ["CORE_PROBE_TEST_TAG"] + del os.environ["GITHUB_TOKEN"] + self.assertEqual(code, 0) + rep = record["variants"][0]["repetitions"][0] + self.assertEqual(rep["logs"][1]["text"], "tag unset") + self.assertTrue(rep["isolation"] == "bwrap" or rep["isolation"].startswith("environment only")) + + +class Sandbox(unittest.TestCase): + """What a core run under bubblewrap can reach.""" + + def setUp(self) -> None: + if not core_probe.sandbox_argv([])[0]: + self.skipTest("no working bwrap on this host") + self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) + self.home = self.tmp / "home" + for secret in (".config/gh/hosts.yml", ".ssh/id_ed25519"): + (self.home / secret).parent.mkdir(parents=True) + (self.home / secret).write_text("secret") + (self.home / ".git-credentials").write_text("https://user:secret@github.com") + self.out = self.tmp / "out" + self.elsewhere = self.tmp / "elsewhere" + self.elsewhere.mkdir() + + def tearDown(self) -> None: + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_credentials_masked_tree_read_only_network_gone(self) -> None: + old_home = os.environ["HOME"] + os.environ["HOME"] = str(self.home) + try: + prefix, isolation = core_probe.sandbox_argv([self.out]) + finally: + os.environ["HOME"] = old_home + self.assertEqual(isolation, "bwrap") + script = (f'cat {self.home}/.config/gh/hosts.yml {self.home}/.ssh/id_ed25519 ' + f'{self.home}/.git-credentials 2>/dev/null; echo ok > {self.out}/w; ' + f'echo no > {self.elsewhere}/w 2>/dev/null; tail -n +3 /proc/net/dev | cut -d: -f1') + proc = subprocess.run(prefix + ["/bin/sh", "-c", script], capture_output=True, text=True, + check=False, env={}) + self.assertNotIn("secret", proc.stdout) + self.assertEqual((self.out / "w").read_text(), "ok\n") + self.assertFalse((self.elsewhere / "w").exists()) + self.assertEqual(proc.stdout.split(), ["lo"]) if __name__ == "__main__":