kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 00:27:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: fetch builds into bare mirrors

Browse Source 
Host git only fetches into bare mirrors no sandbox ever sees, https only, with no user or system config and overrides emptying hooks, fsmonitor, ssh and proxy commands and every filter driver the mirror defines. Each build is a git archive of the commit and of each submodule commit from its own mirror. Old working-tree clones are deleted. The core copied out must be a regular file inside the build folder, opened without following links.
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 12:19:55 +01:00
1 parent fdcf120a7d
commit f5e98bfda6
3 files changed
+279 -108
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/campaign.py
+163 -69
View File
@@ -50,6 +50,7 @@ import os
import re
import shutil
import signal
import stat
import subprocess
import sys
import tarfile
@@ -284,10 +285,18 @@ def child_env(romfetch: bool = False) -> dict[str, str]:
return autorun.safe_env(names)
# Host-side git on a clone of a third-party project: no hook, no fsmonitor,
# no local-file transport, no user or system config.
GIT_SAFE = ("-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false",
"-c", "protocol.file.allow=never", "-c", "protocol.ext.allow=never")
# Host-side git on a third-party project. It only ever touches bare mirrors
# this campaign created: no working tree, never bound into a sandbox. No user
# or system config is read, https is the only transport, and the overrides
# blank every hook, filter, proxy and ssh command a config could name.
GIT_SAFE = (
"-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false",
"-c", "core.sshCommand=", "-c", "core.gitProxy=", "-c", "core.askPass=",
"-c", "credential.helper=", "-c", "protocol.allow=never", "-c", "protocol.https.allow=always",
"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=", "-c", "filter.lfs.process=",
"-c", "filter.lfs.required=false", "-c", "core.attributesFile=/dev/null",
)
MIRROR_URL = re.compile(r"^https://[A-Za-z0-9.-]+(?::\d+)?/[A-Za-z0-9._~/+-]+$")
def host_git(*args: str) -> list[str]:
@@ -296,41 +305,129 @@ def host_git(*args: str) -> list[str]:
def git_env() -> dict[str, str]:
env = child_env()
env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0",
GIT_ALLOW_PROTOCOL="https", GIT_ATTR_NOSYSTEM="1")
return env
def export_tree(src: Path, dest: Path) -> None:
"""The checked-out commit of src and of its submodules, recursively, as
plain files in a fresh dest: git archive read by tarfile, which refuses
absolute paths, .. and links that leave dest."""
def blank_filters(repo: Path) -> list[str]:
"""-c overrides emptying every filter driver the repository's config,
includes followed, defines: git archive runs smudge filters."""
config = repo / "config"
if not config.is_file():
return []
listing = run_group(["git", "config", "--file", str(config), "--includes", "--name-only",
"--get-regexp", r"^filter\."], capture_output=True, env=git_env(),
timeout=60).stdout.decode("utf-8", "replace")
drivers = sorted({line.rsplit(".", 1)[0] for line in listing.splitlines() if line.count(".") >= 2})
out = []
for driver in drivers:
for key in ("smudge", "clean", "process"):
out += ["-c", f"{driver}.{key}="]
out += ["-c", f"{driver}.required=false"]
return out
def git_out(*args: str, timeout: int = 1800) -> bytes:
extra = blank_filters(Path(args[1])) if len(args) > 1 and args[0] == "-C" else []
return run_group(["git", *GIT_SAFE, *extra, *args], capture_output=True, check=True,
env=git_env(), timeout=timeout).stdout
def submodule_url(base: str, url: str) -> str:
"""A .gitmodules URL, relative ones read against the parent's URL."""
if url.startswith(("./", "../")):
parts = base.rstrip("/").removesuffix(".git").split("/")
for piece in url.split("/"):
if piece == "..":
parts.pop()
elif piece not in (".", ""):
parts.append(piece)
url = "/".join(parts)
return url
def extract_tar(blob: bytes, target: Path) -> None:
"""A git archive unpacked into target: no absolute path, no .., no link
that leaves target."""
target.mkdir(parents=True, exist_ok=True)
with tarfile.open(fileobj=io.BytesIO(blob)) as handle:
if hasattr(tarfile, "data_filter"):
handle.extractall(target, filter="data")
return
for member in handle.getmembers():
name = Path(member.name)
link = Path(member.linkname)
if name.is_absolute() or ".." in name.parts or member.islnk() or (
member.issym() and (link.is_absolute() or ".." in link.parts)):
raise Unavailable(f"core: unsafe archive member {member.name}")
handle.extractall(target)
def export_commit(mirror: Callable[[str], Path], url: str, commit: str, dest: Path) -> None:
"""The tree of commit, and of each submodule at the commit the tree
records, as plain files in a fresh dest. Every object comes from a bare
mirror (mirror(url) fetches into it), never from a working tree."""
if dest.exists():
shutil.rmtree(dest)
dest.mkdir(parents=True)
def one(repo: Path, target: Path) -> None:
archive = run_group(host_git("-C", str(repo), "archive", "--format=tar", "HEAD"),
capture_output=True, check=True, env=git_env(), timeout=1800)
with tarfile.open(fileobj=io.BytesIO(archive.stdout)) as handle:
if hasattr(tarfile, "data_filter"):
handle.extractall(target, filter="data")
else:
for member in handle.getmembers():
name = Path(member.name)
if name.is_absolute() or ".." in name.parts or member.islnk() or (
member.issym() and (Path(member.linkname).is_absolute()
or ".." in Path(member.linkname).parts)):
raise Unavailable(f"core: unsafe member {member.name} in {repo}")
handle.extractall(target)
listing = run_group(host_git("-C", str(repo), "ls-files", "--stage", "-z"),
capture_output=True, check=True, env=git_env(), timeout=600)
for entry in listing.stdout.decode("utf-8", "replace").split("\0"):
if entry.startswith("160000 "):
path = entry.split("\t", 1)[1]
if (repo / path / ".git").exists():
one(repo / path, target / path)
def one(repo_url: str, sha: str, target: Path) -> None:
if not re.fullmatch(r"[0-9a-f]{40}", sha):
raise Unavailable(f"core: {sha!r} is not a commit id")
bare = mirror(repo_url)
try:
git_out("-C", str(bare), "cat-file", "-e", f"{sha}^{{commit}}")
except subprocess.CalledProcessError:
git_out("-C", str(bare), "fetch", "--quiet", repo_url, sha)
extract_tar(git_out("-C", str(bare), "archive", "--format=tar", sha), target)
links = {}
for entry in git_out("-C", str(bare), "ls-tree", "-r", "-z", sha).split(b"\0"):
if entry.startswith(b"160000 commit "):
meta, path = entry.split(b"\t", 1)
links[path.decode()] = meta.split()[2].decode()
if not links:
return
try:
config = git_out("-C", str(bare), "config", "--blob", f"{sha}:.gitmodules", "-z",
"--get-regexp", r"^submodule\..*\.(path|url)$").decode()
except subprocess.CalledProcessError as exc:
raise Unavailable(f"core: {repo_url} has submodules and no readable .gitmodules") from exc
paths: dict[str, str] = {}
urls: dict[str, str] = {}
for item in filter(None, config.split("\0")):
key, _, value = item.partition("\n")
name, field_ = key[len("submodule."):].rsplit(".", 1)
(paths if field_ == "path" else urls)[name] = value
for name, path in paths.items():
if path in links and name in urls:
one(submodule_url(repo_url, urls[name]), links[path], target / path)
one(src, dest)
one(url, commit, dest)
def built_core(tree: Path) -> Path:
"""The newest *_libretro.so a build left: a regular file inside the tree,
never a link a build could point at a host file."""
root = tree.resolve()
found = []
for path in tree.rglob("*_libretro.so"):
info = path.lstat()
if stat.S_ISREG(info.st_mode) and path.resolve().is_relative_to(root):
found.append((info.st_mtime, path))
if not found:
raise Unavailable(f"core: the build left no *_libretro.so regular file in {tree}")
return max(found)[1]
def copy_regular(src: Path, dest: Path) -> None:
"""Copy src only when it is a regular file, never through a link."""
fd = os.open(src, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, "rb") as handle:
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
raise Unavailable(f"core: {src} is not a regular file")
dest.parent.mkdir(parents=True, exist_ok=True)
with dest.open("wb") as out:
shutil.copyfileobj(handle, out)
@contextlib.contextmanager
@@ -683,55 +780,52 @@ class Cores:
return Core(built[0].name.removesuffix("_libretro.so"), built[0], file_sha256(built[0]),
repo=source, commit=commit, build_command=command)
def mirror(self, url: str) -> Path:
"""The bare mirror of a repository, created on first use. Only host
git with the overrides of GIT_SAFE ever touches it."""
if not MIRROR_URL.match(url):
raise Unavailable(f"core: {url!r} is not an https repository URL")
bare = self.c.paths.home / "mirrors" / f"{slug(url)}.git"
if not (bare / "HEAD").is_file():
bare.parent.mkdir(parents=True, exist_ok=True)
git_out("clone", "--bare", "--quiet", url, str(bare))
return bare
def compile(self, source: str, commit: str, out: Path) -> tuple[Path, str]:
"""Clone with git (trusted, networked), then run the project's make
under bubblewrap: no network, a read-only tree with the source
writable, the user's credentials masked. Returns the .so and the
command, with how it was isolated."""
src = self.c.paths.home / "src" / slug(source)
"""Fetch the commit into a bare mirror (host git, networked), export
it and its submodules with git archive into a fresh folder, and run
the project's make there under bubblewrap: no network, that folder
the one writable path. Returns the .so and the command, with how it
was isolated."""
log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log"
log.parent.mkdir(parents=True, exist_ok=True)
def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800,
env: dict[str, str] | None = None) -> None:
with log.open("a", encoding="utf-8") as handle:
handle.write(f"$ {' '.join(argv)}\n")
handle.flush()
proc = self.c.host.run(argv, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT,
check=False, timeout=timeout, env=env or git_env())
if proc.returncode != 0:
tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:]
raise Unavailable(f"core: {argv[0]} exited {proc.returncode}: "
f"{' | '.join(tail)} (log {log})")
if core_probe.sandbox_argv([])[1] != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed:
raise Unavailable(f"refused: no sandbox to build {source} in "
f"({core_probe.sandbox_argv([])[1]}), --allow-unsandboxed overrides")
if not (src / ".git").is_dir():
src.parent.mkdir(parents=True, exist_ok=True)
step(host_git("clone", "--quiet", "--no-checkout", source, str(src)))
step(host_git("-C", str(src), "fetch", "--quiet", "origin", commit))
step(host_git("-C", str(src), "checkout", "--quiet", "--force", commit))
step(host_git("-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet"))
# The build runs on an exported tree. The clone and its .git are never
# bound into the sandbox, so a build cannot plant a hook or a config
# line that the next host git command would run.
isolation = core_probe.sandbox_argv([])[1]
if isolation != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed:
raise Unavailable(f"refused: no sandbox to build {source} in ({isolation}), "
f"--allow-unsandboxed overrides")
# Working-tree clones of earlier campaign versions are never reused.
shutil.rmtree(self.c.paths.home / "src", ignore_errors=True)
tree = self.c.paths.home / "build-trees" / f"{slug(source)}-{commit[:12]}"
try:
export_tree(src, tree)
export_commit(self.mirror, source, commit, tree)
except (OSError, subprocess.CalledProcessError, tarfile.TarError) as exc:
raise Unavailable(f"core: exporting {commit[:12]} of {source} failed: {exc}") from exc
makefile = "Makefile.libretro" if (tree / "Makefile.libretro").is_file() else "Makefile"
jobs = str(max(1, (os.cpu_count() or 2) // 2))
command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"]
prefix, isolation = core_probe.sandbox_argv([tree])
step(prefix + command, cwd=tree, timeout=BUILD_TIMEOUT, env=child_env())
built = sorted(tree.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime)
if not built:
raise Unavailable(f"core: {' '.join(command)} produced no *_libretro.so (log {log})")
out.mkdir(parents=True, exist_ok=True)
shutil.copy2(built[-1], out / built[-1].name)
with log.open("a", encoding="utf-8") as handle:
handle.write(f"$ {' '.join(command)} in {tree} ({isolation})\n")
handle.flush()
proc = self.c.host.run(prefix + command, cwd=tree, stdout=handle, stderr=subprocess.STDOUT,
check=False, timeout=BUILD_TIMEOUT, env=child_env())
if proc.returncode != 0:
tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:]
raise Unavailable(f"core: make exited {proc.returncode}: {' | '.join(tail)} (log {log})")
built = built_core(tree)
copy_regular(built, out / built.name)
shutil.rmtree(tree, ignore_errors=True)
return out / built[-1].name, f"{' '.join(command)} on git archive (isolation: {isolation})"
return out / built.name, f"{' '.join(command)} on git archive (isolation: {isolation})"
# --------------------------------------------------------------------------
tests/test_campaign.py
+115 -38
View File
@@ -470,66 +470,143 @@ class FailClosed(Workspace):
@unittest.skipUnless(shutil.which("git") and shutil.which("make"), "no git or make")
class BuildIsolation(unittest.TestCase):
"""A build that writes into .git must not reach the next host git command."""
"""Builds run on an archive exported from bare mirrors that only host git
touches: a build cannot reach the mirror, and host git runs nothing a
repository's config or attributes name."""
MAIN = "https://example.invalid/team/main.git"
SUB = "https://example.invalid/team/sub.git"
def setUp(self) -> None:
TMP.mkdir(parents=True, exist_ok=True)
self.tmp = Path(tempfile.mkdtemp(dir=TMP))
self.addCleanup(shutil.rmtree, self.tmp, True)
self.src = self.tmp / "src"
self.pwned = self.tmp / "PWNED"
hook = f"#!/bin/sh\ntouch {self.pwned}\n"
# A Makefile that tries to plant a hook and a config line in the clone.
sub = self.repo("sub", {"sub.c": "int x;\n"})
self.sub_sha = self.git(sub, "rev-parse", "HEAD").strip()
self.mirrors = {self.SUB: self.bare(sub, "sub")}
main_mirror = self.tmp / "mirrors" / "main.git"
# A Makefile that tries to plant a hook and a config line in the mirror.
makefile = (
"all:\n"
f"\t-mkdir -p {self.src}/.git/hooks ../.git/hooks\n"
f"\t-printf '{hook.replace(chr(10), '\\\\n')}' > {self.src}/.git/hooks/post-checkout\n"
f"\t-printf '[core]\\n\\tfsmonitor = touch {self.pwned}\\n' >> {self.src}/.git/config\n"
f"\t-mkdir -p {main_mirror}/hooks\n"
f"\t-echo 'touch {self.pwned}' > {main_mirror}/hooks/post-checkout\n"
f"\t-printf '[core]\\n\\tfsmonitor = touch {self.pwned}\\n' >> {main_mirror}/config\n"
"\ttouch fake_libretro.so\n")
git = ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", str(self.src)]
subprocess.run(["git", "init", "-q", str(self.src)], check=True)
(self.src / "Makefile").write_text(makefile)
(self.src / "a.txt").write_text("one")
subprocess.run([*git, "add", "Makefile", "a.txt"], check=True)
subprocess.run([*git, "commit", "-q", "-m", "one"], check=True)
(self.src / "a.txt").write_text("two")
subprocess.run([*git, "commit", "-q", "-am", "two"], check=True)
self.config = (self.src / ".git" / "config").read_text()
main = self.repo("main", {"Makefile": makefile, ".gitmodules":
'[submodule "lib"]\n\tpath = lib\n\turl = ../sub.git\n'})
self.git(main, "update-index", "--add", "--cacheinfo", f"160000,{self.sub_sha},lib")
self.git(main, "commit", "-q", "-m", "submodule")
self.commit = self.git(main, "rev-parse", "HEAD").strip()
self.mirrors[self.MAIN] = self.bare(main, "main")
self.config = (main_mirror / "config").read_text()
def test_the_build_runs_on_an_export_without_git(self) -> None:
prefix, isolation = campaign.core_probe.sandbox_argv([self.tmp / "tree"])
def git(self, repo: Path, *args: str) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", str(repo), *args],
check=True, capture_output=True, text=True).stdout
def repo(self, name: str, files: dict[str, str]) -> Path:
path = self.tmp / "work" / name
subprocess.run(["git", "init", "-q", str(path)], check=True)
for rel, text in files.items():
(path / rel).write_text(text)
self.git(path, "add", *files)
self.git(path, "commit", "-q", "-m", "one")
return path
def bare(self, work: Path, name: str) -> Path:
path = self.tmp / "mirrors" / f"{name}.git"
subprocess.run(["git", "clone", "-q", "--bare", str(work), str(path)], check=True)
return path
def test_the_export_holds_the_tree_and_its_submodule_and_no_git(self) -> None:
tree = self.tmp / "tree"
campaign.export_commit(self.mirrors.__getitem__, self.MAIN, self.commit, tree)
self.assertEqual((tree / "lib" / "sub.c").read_text(), "int x;\n")
self.assertTrue((tree / "Makefile").is_file())
self.assertEqual([p for p in tree.rglob(".git")], [])
def test_a_build_cannot_reach_the_mirror(self) -> None:
tree = self.tmp / "tree"
prefix, isolation = campaign.core_probe.sandbox_argv([tree])
if isolation != campaign.core_probe.SANDBOX:
self.skipTest("no working bwrap on this host")
tree = self.tmp / "tree"
campaign.export_tree(self.src, tree)
self.assertFalse((tree / ".git").exists())
self.assertEqual((tree / "a.txt").read_text(), "two")
campaign.export_commit(self.mirrors.__getitem__, self.MAIN, self.commit, tree)
proc = subprocess.run(prefix + ["make"], cwd=tree, capture_output=True, text=True,
check=False, env=campaign.child_env())
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertTrue((tree / "fake_libretro.so").exists())
self.assertFalse((self.src / ".git" / "hooks" / "post-checkout").exists())
self.assertEqual((self.src / ".git" / "config").read_text(), self.config)
subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"),
check=True, env=campaign.git_env())
mirror = self.mirrors[self.MAIN]
self.assertFalse((mirror / "hooks" / "post-checkout").exists())
self.assertEqual((mirror / "config").read_text(), self.config)
campaign.export_commit(self.mirrors.__getitem__, self.MAIN, self.commit, self.tmp / "again")
self.assertFalse(self.pwned.exists())
def test_host_git_runs_no_planted_hook_or_fsmonitor(self) -> None:
hook = self.src / ".git" / "hooks" / "post-checkout"
def test_host_git_runs_nothing_a_repository_config_names(self) -> None:
mirror = self.mirrors[self.MAIN]
hook = mirror / "hooks" / "reference-transaction"
hook.write_text(f"#!/bin/sh\ntouch {self.pwned}\n")
hook.chmod(0o755)
with (self.src / ".git" / "config").open("a") as handle:
handle.write(f"[core]\n\tfsmonitor = touch {self.pwned}\n")
subprocess.run(campaign.host_git("-C", str(self.src), "checkout", "-q", "HEAD~1"),
check=True, env=campaign.git_env())
subprocess.run(campaign.host_git("-C", str(self.src), "status", "--short"),
check=True, env=campaign.git_env(), capture_output=True)
with (mirror / "config").open("a") as handle:
handle.write(f"[core]\n\tfsmonitor = touch {self.pwned}\n\tsshCommand = touch {self.pwned}\n"
f'[filter "x"]\n\tsmudge = touch {self.pwned}\n')
(mirror / "info").mkdir(exist_ok=True)
(mirror / "info" / "attributes").write_text("* filter=x\n")
campaign.export_commit(self.mirrors.__getitem__, self.MAIN, self.commit, self.tmp / "tree")
campaign.git_out("-C", str(mirror), "update-ref", "refs/x", self.commit)
self.assertFalse(self.pwned.exists())
# The same checkout with plain git does run the hook: the test plants a live one.
subprocess.run(["git", "-C", str(self.src), "checkout", "-q", "HEAD@{1}"], check=True,
capture_output=True)
# Plain git on the same mirror runs the planted hook: the test plants a live one.
subprocess.run(["git", "-C", str(mirror), "update-ref", "refs/y", self.commit], check=True)
self.assertTrue(self.pwned.exists())
def test_only_https_mirrors(self) -> None:
c = campaign.Campaign(campaign.Paths(root=self.tmp, home=self.tmp / "home", content=self.tmp,
db=self.tmp / "db.json", content_file=campaign.CONTENT_FILE),
campaign.Settings(echo=False), FakeHost(), lock_path=self.tmp / "l")
for url in ("file:///etc", "/home/x/repo", "ext::sh -c touch", "ssh://h/r", "git@h:r"):
with self.subTest(url=url), self.assertRaises(campaign.Unavailable):
c.cores.mirror(url)
self.assertEqual(campaign.submodule_url(self.MAIN, "../sub.git"), self.SUB)
self.assertEqual(campaign.git_env()["GIT_ALLOW_PROTOCOL"], "https")
def test_a_whole_build_wipes_old_clones_and_copies_a_regular_file(self) -> None:
if campaign.core_probe.sandbox_argv([])[1] != campaign.core_probe.SANDBOX:
self.skipTest("no working bwrap on this host")
home = self.tmp / "home"
(home / "src" / "old" / ".git").mkdir(parents=True)
c = campaign.Campaign(campaign.Paths(root=self.tmp, home=home, content=self.tmp,
db=self.tmp / "db.json", content_file=campaign.CONTENT_FILE),
campaign.Settings(echo=False), campaign.Host(), lock_path=self.tmp / "l")
c.cores.mirror = self.mirrors.__getitem__
so, command = c.cores.compile(self.MAIN, self.commit, self.tmp / "out")
self.assertEqual(so, self.tmp / "out" / "fake_libretro.so")
self.assertIn("isolation: bwrap", command)
self.assertFalse((home / "src").exists())
class BuiltCore(unittest.TestCase):
def setUp(self) -> None:
TMP.mkdir(parents=True, exist_ok=True)
self.tree = Path(tempfile.mkdtemp(dir=TMP))
self.addCleanup(shutil.rmtree, self.tree, True)
self.secret = self.tree.parent / f"{self.tree.name}-host-file"
self.secret.write_text("host secret")
self.addCleanup(self.secret.unlink)
def test_a_link_is_never_taken_for_the_core(self) -> None:
(self.tree / "a_libretro.so").symlink_to(self.secret)
with self.assertRaises(campaign.Unavailable):
campaign.built_core(self.tree)
(self.tree / "sub").mkdir()
(self.tree / "sub" / "b_libretro.so").write_bytes(b"\x7fELF")
self.assertEqual(campaign.built_core(self.tree), self.tree / "sub" / "b_libretro.so")
def test_copy_refuses_a_link(self) -> None:
link = self.tree / "c_libretro.so"
link.symlink_to(self.secret)
with self.assertRaises(OSError):
campaign.copy_regular(link, self.tree / "out.so")
self.assertFalse((self.tree / "out.so").exists())
class ProbeRuns(Workspace):
def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess:
wiki/tools.md
+1 -1
View File
@@ -955,7 +955,7 @@ python scripts/campaign.py install-timer # the autorun units, the campaign's
| Step | What happens |
|------|--------------|
| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. git clones with the allowlisted environment, no user or system config, and `core.hooksPath=/dev/null`, `core.fsmonitor=false` and the `file` and `ext` transports refused. The commit and its submodules are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path: the clone and its `.git` are never inside, so a build cannot plant a hook or a config line for the next host git command. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` |
| Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. Host git only fetches into bare mirrors (`.cache/campaign/mirrors/`), one per repository, submodules included, that it alone touches: no working tree, never bound into a sandbox, and earlier working-tree clones are deleted. It runs with no user or system config, https as the only transport, and `-c` overrides emptying hooks, `core.fsmonitor`, `core.sshCommand`, `core.gitProxy`, credential helpers and every filter driver the mirror's config defines. The commit and each submodule at the commit its tree records are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path. The core copied out is the newest `*_libretro.so` that is a regular file inside that folder, opened without following links. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` |
| Content | Each system's games come from `tests/campaign/content.yml`, by catalog name and with the reason each was chosen. They are read from `.cache/test-content/<romfetch system>/<name>/`, each file checked against its `identity.json`. Missing content is fetched with `romfetch get <system> <name> --json` when romfetch is on `PATH`, in `RETROBIOS_ROMFETCH` or given with `--romfetch`. Otherwise the checks are `pending: content`, with romfetch's exit code and error code when it failed. Content never enters git, and a record keeps only its identity |
| Checks | `files` stages every declared file the collection holds and reads which ones the core opens (strace). `boot` runs the game with no file and with the declared one (`boot_file:`, else the entry of the game's region). An entry the `files` run left unread is staged `alone`, then where the core looked for it (`path`) and under the names it looked for (`alias`). `variants` runs each `.variants/` copy of a file the core opened. `bench` times the game, warm-up excluded, 5 repetitions |
| Rules | `campaign_rules.py`, a fixed table of pure functions (below). A field the table settles is written into the profile, with `evidence:` naming the record ids. Anything else is a line of `measurements/findings.jsonl` |