chore: merge engine fixes, third pass

This commit is contained in:
Abdessamad Derraz committed 2026-11-06 05:06:59 +01:00
commit 7a98d73d79
37 files changed
+985 -186

No files matched your search

+55 -1
View File
@@ -23,6 +23,7 @@ from common import ( # noqa: E402
artifact_lock,
hold_artifact_lock,
hold_repository_outputs,
publish_hold,
)
@@ -218,7 +219,7 @@ class PipelineHoldsItsFixedOutputs(unittest.TestCase):
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=120,
)
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
self.assertIn("another pipeline run is writing database.json", proc.stderr)
self.assertIn("another run is writing database.json", proc.stderr)
self.assertNotIn("generate database", proc.stdout.lower())
def test_two_holds_of_the_repository_exclude_each_other(self):
@@ -228,6 +229,59 @@ class PipelineHoldsItsFixedOutputs(unittest.TestCase):
pass
class StandaloneWritersHonourTheRepositoryHold(unittest.TestCase):
"""Only pipeline runs took the repository's lock: generate_db.py beside
a pipeline replaced database.json between its verify and its packs, and
a manifest run on install/ failed the pipeline's own manifest step."""
def _beside_a_pipeline(self, argv: list[str]) -> subprocess.CompletedProcess:
env = {k: v for k, v in os.environ.items() if k != "RETROBIOS_HELD_ARTIFACT_DIR"}
return subprocess.run(
[sys.executable, *argv], capture_output=True, text=True,
cwd=str(REPO_ROOT), timeout=300, env=env, check=False,
)
def test_a_writer_of_a_fixed_output_is_refused(self):
with hold_repository_outputs(str(REPO_ROOT)):
for argv in (
["scripts/generate_db.py", "--bios-dir", "bios", "--output", "database.json"],
["scripts/generate_pack.py", "--all", "--manifest", "--offline",
"--output-dir", "install"],
["scripts/generate_games_index.py", "--quiet"],
):
with self.subTest(script=argv[0]):
proc = self._beside_a_pipeline(argv)
self.assertEqual(proc.returncode, 1, proc.stdout[-500:] + proc.stderr[-500:])
self.assertIn("another run is writing database.json", proc.stderr)
self.assertNotIn("Traceback", proc.stderr)
def _child(self, target: str, env: dict) -> subprocess.CompletedProcess:
code = (
"import sys; sys.path.insert(0, 'scripts')\n"
"from common import artifact_lock, repository_output_lock\n"
f"with repository_output_lock({target!r}), artifact_lock({self.scratch!r}):\n"
" pass\n"
)
return subprocess.run([sys.executable, "-c", code], cwd=str(REPO_ROOT), env=env,
capture_output=True, text=True, timeout=60, check=False)
def test_a_step_the_pipeline_spawns_passes(self):
"""A pipeline holds the root and its --output-dir, and its steps
write both: the environment names the two, os.pathsep apart."""
self.scratch = _tempdir(self)
with hold_repository_outputs(str(REPO_ROOT)), publish_hold(str(REPO_ROOT)), \
hold_artifact_lock(self.scratch):
proc = self._child(str(REPO_ROOT / "install" / "games"), dict(os.environ))
self.assertEqual(proc.returncode, 0, proc.stderr)
def test_a_scratch_output_takes_nothing(self):
self.scratch = _tempdir(self)
env = {k: v for k, v in os.environ.items() if k != "RETROBIOS_HELD_ARTIFACT_DIR"}
with hold_repository_outputs(str(REPO_ROOT)):
proc = self._child(os.path.join(self.scratch, "database.json"), env)
self.assertEqual(proc.returncode, 0, proc.stderr)
class HeldLockTest(unittest.TestCase):
"""pipeline.py took the lock step by step: between the pack build and the
integrity check dist/ was free, another run's purge emptied it, and the
+38
View File
@@ -296,5 +296,43 @@ class RestoreSkipsDownloadsInProgress(unittest.TestCase):
self.assertIn(str(cache / "done.bin"), index.values())
class ATerminatedBuildLeavesNoScratch(unittest.TestCase):
"""`timeout 120 generate_pack.py --platform RetroArch` left a 3.4 GB
hidden .partial in dist/: SIGTERM ended the process without running the
cleanup a failed write runs."""
CHILD = (
"import sys, time\n"
"sys.path.insert(0, 'scripts')\n"
"from common import run_cli\n"
"from generate_pack import _writing_pack\n"
"def main():\n"
" with _writing_pack(sys.argv[1]) as zf:\n"
" zf.writestr('a.bin', b'x')\n"
" print('writing', flush=True)\n"
" time.sleep(60)\n"
"run_cli(main)\n"
)
def test_sigterm_removes_the_partial_pack(self):
import signal # noqa: PLC0415
import subprocess # noqa: PLC0415
if not hasattr(signal, "SIGTERM") or os.name != "posix":
self.skipTest("POSIX signals")
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
pack = Path(tmp) / "Demo_BIOS_Pack.zip"
child = subprocess.Popen(
[sys.executable, "-c", self.CHILD, str(pack)], cwd=REPO_ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
)
self.assertEqual(child.stdout.readline().strip(), "writing")
self.assertTrue(list(Path(tmp).glob(".*.partial")))
child.send_signal(signal.SIGTERM)
child.communicate(timeout=60)
self.assertEqual(child.returncode, 128 + signal.SIGTERM)
self.assertEqual(sorted(p.name for p in Path(tmp).iterdir()), [])
if __name__ == "__main__":
unittest.main()
+105 -17
View File
@@ -118,6 +118,24 @@ class ReadmeRegressions(unittest.TestCase):
self.assertEqual(proc.returncode, 1)
self.assertIn("needs Python 3.9", proc.stderr)
def test_the_readme_build_command_names_the_floor(self):
"""The README's first command is generate_pack.py. Under 3.8 it
stopped on a raw SyntaxError at a parenthesized `with`."""
import shutil # noqa: PLC0415
python38 = shutil.which("python3.8")
if python38 is None:
if os.environ.get("CI"):
self.fail("CI must provide python3.8")
self.skipTest("no python3.8 on PATH")
proc = subprocess.run(
[python38, "-I", "scripts/generate_pack.py", "--help"],
cwd=ROOT, capture_output=True, text=True, timeout=60, check=False,
)
self.assertEqual(proc.returncode, 1)
self.assertNotIn("SyntaxError", proc.stderr)
self.assertIn("generate_pack.py needs Python 3.9", proc.stderr)
def test_the_floor_interpreter_imports_every_script(self):
"""Every module under scripts/ is loaded by a real interpreter of the
floor version, as the installer's 3.8 floor is proven by running it.
@@ -948,6 +966,20 @@ class CatalogRatioRegressions(unittest.TestCase):
class PipelineRegressions(unittest.TestCase):
def test_integrity_checks_the_variants_step_4_built(self):
"""Step 6 ran --verify-packs without --all-variants: the five
narrowed packs were never opened and an empty _Platform passed."""
import argparse # noqa: PLC0415
def command(**flags) -> list[str]:
base = {"output_dir": "dist", "include_archived": False,
"target": None, "all_variants": False}
return pipeline.integrity_command(argparse.Namespace(**{**base, **flags}))
self.assertIn("--all-variants", command(all_variants=True))
self.assertNotIn("--all-variants", command())
self.assertEqual(command(target="switch")[-2:], ["--target", "switch"])
def test_pack_parser_removes_source_metadata(self):
output = "\n".join(
[
@@ -2734,6 +2766,10 @@ class TheProjectInstallsWithPip(unittest.TestCase):
import importlib.util # noqa: PLC0415
if importlib.util.find_spec("setuptools") is None:
# The dev extra and CI install it: skipped there, the guard held
# nothing on any run.
if os.environ.get("CI"):
self.fail("CI must install setuptools")
self.skipTest("setuptools is not installed")
program = (
"from setuptools.dist import Distribution\n"
@@ -2808,27 +2844,35 @@ class ACheckThatCannotAnswerDoesNotPass(unittest.TestCase):
self.assertEqual(unsatisfied, ["bios/absent.bin"])
def test_an_unreachable_buildbot_is_not_a_fresh_verdict(self):
source = (ROOT / "scripts" / "check_buildbot_system.py").read_text(
encoding="utf-8"
)
self.assertIn(
'if report.get("error") or pending(report):',
source.split("def main(")[-1],
"main() ignores the error the report carries",
)
"""Every HEAD unanswered printed "16 unknown" and exited 0, so the
online pipeline printed check_buildbot OK."""
sys.path.insert(0, str(ROOT / "scripts"))
import check_buildbot_system as cbs # noqa: PLC0415
registry = str(ROOT / "platforms" / "_data_dirs.yml")
tracked = cbs.load_tracked_entries(registry)
self.assertTrue(tracked)
cases = {
"index unreachable": mock.patch.object(
cbs, "fetch_index", side_effect=OSError("down")),
"every HEAD unanswered": mock.patch.object(
cbs, "fetch_index", return_value=set(tracked)),
}
for label, index in cases.items():
with self.subTest(label), index, \
mock.patch.object(cbs, "get_remote_etag", return_value=None), \
mock.patch.object(sys, "argv", ["check_buildbot_system.py",
"--registry", registry]), \
contextlib.redirect_stdout(io.StringIO()), \
self.assertRaises(SystemExit) as stop:
cbs.main()
self.assertEqual(stop.exception.code, 1)
def test_a_missing_profile_directory_says_so(self):
import io as _io
from scripts import common as _common
stderr = _io.StringIO()
with contextlib.redirect_stderr(stderr):
profiles = _common.load_emulator_profiles(
str(ROOT / "no-such-emulator-dir")
)
self.assertEqual(profiles, {})
self.assertIn("no emulator profile directory", stderr.getvalue())
with self.assertRaisesRegex(_common.InputMissing, "no emulator profile directory"):
_common.load_emulator_profiles(str(ROOT / "no-such-emulator-dir"))
class TestEntryPointsRunEveryClass(unittest.TestCase):
@@ -4145,6 +4189,8 @@ class EveryToolRefusesAFlagItWouldDrop(unittest.TestCase):
["--offline", "--all-variants", "--source", "platform"],
["--offline", "--all-variants", "--source", "full"],
["--offline", "--skip-packs", "--source", "full"],
["--offline", "--source", "truth", "--full-narrowings"],
["--offline", "--source", "platform", "--full-narrowings"],
],
}
@@ -7138,5 +7184,47 @@ class OpenMsxSystemRomsAreFoundBySha1(unittest.TestCase):
self.assertEqual(named, [])
class TupleAnnotationsMatchTheirReturns(unittest.TestCase):
"""`_by_affinity.score` gained a fifth sort key while its annotation kept
four, so the signature misread the order that sends C-Dogs'
gamecontrollerdb.txt behind a neutral copy."""
@staticmethod
def _own_returns(fn: ast.AST):
stack = list(ast.iter_child_nodes(fn))
while stack:
node = stack.pop()
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
continue
if isinstance(node, ast.Return):
yield node
stack.extend(ast.iter_child_nodes(node))
def test_a_fixed_tuple_annotation_counts_its_returned_values(self):
sources = sorted((ROOT / "scripts").rglob("*.py")) + [ROOT / "install.py"]
mismatched = []
for path in sources:
for fn in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
hint = fn.returns
if not (isinstance(hint, ast.Subscript) and isinstance(hint.value, ast.Name)
and hint.value.id == "tuple"):
continue
declared = hint.slice.elts if isinstance(hint.slice, ast.Tuple) else [hint.slice]
if any(isinstance(e, ast.Constant) and e.value is Ellipsis for e in declared):
continue
for ret in self._own_returns(fn):
value = ret.value
if (isinstance(value, ast.Tuple)
and not any(isinstance(e, ast.Starred) for e in value.elts)
and len(value.elts) != len(declared)):
mismatched.append(
f"{path.relative_to(ROOT)}:{ret.lineno} "
f"{len(value.elts)} returned, {len(declared)} declared"
)
self.assertEqual(mismatched, [])
if __name__ == "__main__":
unittest.main()
+25
View File
@@ -165,5 +165,30 @@ class AmberelecCarriesTheMachines(unittest.TestCase):
)
class TheCountLineKeepsTreesOutOfTheBaseline(unittest.TestCase):
"""MyMinUI declares no file, and its pack line read "6944 baseline": the
fbneo trees its cores read were counted as the platform's list."""
def test_a_platform_declaring_nothing_has_no_baseline(self) -> None:
import contextlib
import io
previous = os.getcwd()
os.chdir(REPO_ROOT)
self.addCleanup(os.chdir, previous)
db = common.load_database("database.json")
record = builder.PackRecord()
out = io.StringIO()
with contextlib.redirect_stdout(out):
builder.generate_pack(
"myminui", "platforms", db, "bios", "tmp", offline=True, record=record,
data_registry=common.load_data_dir_registry("platforms"),
)
line = next(l for l in out.getvalue().splitlines() if "files packed" in l)
self.assertIn("(0 baseline + ", line)
if record.trees:
self.assertIn(f"+ {len(record.trees)} in core data trees + ", line)
if __name__ == "__main__":
unittest.main()
+32
View File
@@ -95,6 +95,38 @@ class CoverageCache(unittest.TestCase):
self._run()
self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1)
def test_a_purge_racing_another_generator_does_not_fail(self):
"""generate_site and generate_readme purge the same directory: one
unlinking a pickle the other had listed ended on FileNotFoundError."""
cache_dir = self.work / "cache"
cache_dir.mkdir()
ghost = cache_dir / "gone.pickle"
real_glob = Path.glob
def listing_a_vanished_file(path, pattern):
found = list(real_glob(path, pattern))
return [*found, ghost] if path == cache_dir else found
with mock.patch.object(Path, "glob", listing_a_vanished_file):
coverages = self._run()
self.assertEqual(sorted(coverages), ["alpha", "beta"])
self.assertEqual(len(list(cache_dir.glob("*.pickle"))), 1)
def test_a_missing_input_directory_stops_the_run(self):
"""A missing --emulators-dir skipped every platform as if it had no
YAML, then replaced the cache with the empty result."""
self._run()
kept = sorted((self.work / "cache").glob("*.pickle"))
def missing(*_args, **_kwargs):
raise generate_readme.InputMissing("no emulator profile directory at nosuch")
with mock.patch.object(generate_readme, "compute_coverage", missing), \
mock.patch.object(generate_readme, "_coverage_inputs", return_value="other"), \
self.assertRaises(generate_readme.InputMissing):
self._run()
self.assertEqual(sorted((self.work / "cache").glob("*.pickle")), kept)
if __name__ == "__main__":
unittest.main()
+30
View File
@@ -31,5 +31,35 @@ class FinalizeByExactName(unittest.TestCase):
self.assertTrue(ok)
class APackFromAnotherRunIsLeftAlone(unittest.TestCase):
"""A one-platform build verified, recompressed and gave a new manifest to
every pack in the directory: MiSTer's lost its pack_systems fields after a
MinUI run, and a bizhawk run rewrote 52 GB of packs it never built."""
def test_a_second_platform_does_not_rewrite_the_first(self):
import json
import subprocess
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
def build(platform: str) -> str:
proc = subprocess.run(
[sys.executable, "scripts/generate_pack.py", "--platform", platform,
"--offline", "--output-dir", tmp],
cwd=REPO_ROOT, capture_output=True, text=True, timeout=600, check=False,
)
self.assertEqual(proc.returncode, 0, proc.stdout[-2000:] + proc.stderr[-2000:])
return proc.stdout
build("misterfpga")
(first,) = Path(tmp).glob("MiSTer*_BIOS_Pack.zip")
before = first.read_bytes()
output = build("minui")
self.assertNotIn(first.name, output)
self.assertEqual(first.read_bytes(), before)
with zipfile.ZipFile(first) as zf:
manifest = json.loads(zf.read("manifest.json"))
self.assertIn("install_list", manifest)
if __name__ == "__main__":
unittest.main()
+23 -1
View File
@@ -182,6 +182,28 @@ class EveryNarrowingIsCompared(unittest.TestCase):
"compare the new narrowing in check_narrowed_packs, "
"or write in NARROWINGS_NOT_COMPARED why it is not")
def test_each_emulator_pack_narrowing_is_compared(self):
"""check_emulator_packs built every pack without --region or
--required-only, while the table above counted both as compared:
it describes the platform packs only."""
import inspect
import pipeline
from generate_pack import _narrowings, generate_emulator_pack
taken = (set(inspect.signature(_narrowings).parameters)
& set(inspect.signature(generate_emulator_pack).parameters))
self.assertEqual(taken, set(pipeline.EMULATOR_NARROWINGS_COMPARED))
profile = {"files": [{"name": "a.bin", "region": "japan"},
{"name": "b.bin", "region": ["north-america", "europe"]}]}
used = {key for _label, narrowing in pipeline.emulator_variants([profile], False)
for key in narrowing}
# standalone is the loop around the variants, both builds of each profile.
self.assertEqual(used, pipeline.EMULATOR_NARROWINGS_COMPARED - {"standalone"})
regions = [n["regions"] for _l, n in pipeline.emulator_variants([profile], True)
if "regions" in n]
self.assertEqual(len(regions), 3)
def test_the_variants_reach_every_compared_dimension(self):
import pipeline
from common import load_emulator_profiles, load_platform_config
@@ -246,7 +268,7 @@ class ThePipelineComparesPaths(unittest.TestCase):
def test_the_run_compares_every_emulator_pack(self):
source = (REPO_ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
self.assertIn("ok = check_pack_destinations(destinations_file, pack_output) and ok", source)
self.assertIn("ok = check_emulator_packs() and ok", source)
self.assertIn("ok = check_emulator_packs(full=args.full_narrowings) and ok", source)
self.assertIn('"--destinations", destinations_file', source)
+25
View File
@@ -93,6 +93,31 @@ class RefreshConcurrency(unittest.TestCase):
)
self.assertEqual(seen, [True])
def test_a_current_tree_is_not_waited_for(self):
"""A pipeline reads every tree for its whole run. A build beside it
waited on the exclusive lock before asking the remote, which would
have answered that nothing changed."""
import fcntl # noqa: PLC0415
self.cache.mkdir(parents=True)
rdd._record_version("pak", {"sha": "e1"}, self.versions)
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
"local_cache": str(self.cache)}
result: list = []
with open(self.cache.with_name(".pak.lock"), "a") as reader, \
mock.patch.object(rdd, "_get_remote_etag", return_value="e1"):
fcntl.flock(reader, fcntl.LOCK_SH)
worker = threading.Thread(target=lambda: result.append(
rdd.refresh_entry("pak", entry, versions_path=self.versions)
), daemon=True)
worker.start()
worker.join(timeout=20)
finished = not worker.is_alive()
fcntl.flock(reader, fcntl.LOCK_UN)
worker.join(timeout=20)
self.assertTrue(finished, "the refresh waited for a reader")
self.assertEqual(result, [False])
def test_version_file_is_never_left_truncated(self):
Path(self.versions).parent.mkdir(parents=True)
Path(self.versions).write_text(json.dumps({"keep": {"sha": "1"}}))
+213
View File
@@ -262,6 +262,219 @@ class AMissingBiosDirIsRefused(unittest.TestCase):
self.assertFalse(Path(out).exists())
def _scripts_taking(flag: str) -> list[Path]:
pattern = re.compile(r'add_argument\(\s*"%s"' % re.escape(flag))
return [
path for path in sorted((REPO_ROOT / "scripts").glob("*.py"))
+ sorted((REPO_ROOT / "scripts" / "scraper").glob("*.py"))
if pattern.search(path.read_text(encoding="utf-8"))
]
def _command(path: Path) -> list[str]:
if path.parent.name == "scraper":
return [sys.executable, "-m", f"scripts.scraper.{path.stem}"]
return [sys.executable, f"scripts/{path.name}"]
class AMissingProfileDirIsRefused(unittest.TestCase):
"""A mistyped --emulators-dir read as "no emulator": generate_pack built
a pack without its core files marked OK, verify --strict passed, and
sitesources wrote {} over source-paths.json, every one exiting 0."""
EXTRA = {
"arcade_parent_importer": ["--all", "--offline"],
"fileless_audit": ["--all"],
"generate_pack": ["--platform", "retroarch", "--system", "atari-lynx",
"--offline", "--output-dir", "{out}"],
"generate_site": ["--docs-dir", "{out}"],
"generate_truth": ["--all", "--output-dir", "{out}"],
"mame_ref_audit": ["mame"],
"profile_sync": ["--all", "--triage"],
"romset_dat_importer": ["--source", "mame", "--pack", "{dat}", "--output", "{out}/r.json"],
"sitesources": ["--cache-dir", "{cache}", "--output", "{out}/source-paths.json"],
"validate_pr": ["bios/Sony/PlayStation/scph5501.bin"],
"verify": ["--platform", "retroarch", "--strict"],
}
def test_every_script_taking_an_emulators_dir_refuses_a_missing_one(self):
scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
self.addCleanup(scratch.cleanup)
root = Path(scratch.name)
(root / "cache").mkdir()
(root / "empty.dat").write_bytes(b"")
values = {"out": str(root / "out"), "cache": str(root / "cache"),
"dat": str(root / "empty.dat")}
taking = _scripts_taking("--emulators-dir")
self.assertGreater(len(taking), 15)
tracked = ["git", "diff", "--name-only"]
before = subprocess.run(tracked, cwd=REPO_ROOT, capture_output=True, text=True).stdout
for path in taking:
extra = [arg.format(**values) for arg in self.EXTRA.get(path.stem, [])]
with self.subTest(script=path.stem):
proc = subprocess.run(
[*_command(path), "--emulators-dir", "nosuch", *extra],
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600,
check=False, env={**os.environ, "RETROBIOS_OFFLINE": "1"},
)
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
self.assertRegex(proc.stderr, r"ERROR: .*nosuch")
written = [
str(p.relative_to(root)) for p in (root / "out").rglob("*")
if p.is_file() and p.name != ".lock"
]
self.assertEqual(written, [])
# Read as empty, generate_site rewrote published-parts.json and the
# FAQ, and sitesources source-paths.json.
after = subprocess.run(tracked, cwd=REPO_ROOT, capture_output=True, text=True).stdout
self.assertEqual(after, before)
class AMissingPlatformsDirIsRefused(unittest.TestCase):
"""`verify.py --platforms-dir noplat` ended on a FileNotFoundError at the
bottom of a stack trace, and validate_pr read it as no platform at all
and exited 0."""
EXTRA = {
"auto_fetch": ["--all", "--dry-run"],
"check_release": ["--tag", "v1"],
"diff_truth": ["--all"],
"export_native": ["--all", "--output-dir", "{out}"],
"generate_pack": ["--platform", "retroarch", "--offline", "--output-dir", "{out}"],
"generate_site": ["--docs-dir", "{out}"],
"generate_truth": ["--all", "--output-dir", "{out}"],
"release_notes": ["--whats-new", "{notes}", "--output", "{out}/notes.md"],
"release_record": ["{out}", "--tag", "v1"],
"romset_dat_importer": ["--source", "mame", "--pack", "{dat}", "--output", "{out}/r.json"],
"validate_pr": ["bios/Sony/PlayStation/scph5501.bin"],
"validate_site": ["--site-dir", "{out}"],
"verify": ["--platform", "retroarch"],
}
def test_every_script_taking_a_platforms_dir_refuses_a_missing_one(self):
scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
self.addCleanup(scratch.cleanup)
root = Path(scratch.name)
(root / "empty.dat").write_bytes(b"")
(root / "notes.md").write_text("Notes.\n")
values = {"out": str(root / "out"), "dat": str(root / "empty.dat"),
"notes": str(root / "notes.md")}
taking = _scripts_taking("--platforms-dir")
self.assertGreater(len(taking), 15)
for path in taking:
extra = [arg.format(**values) for arg in self.EXTRA.get(path.stem, [])]
with self.subTest(script=path.stem):
proc = subprocess.run(
[*_command(path), "--platforms-dir", "nosuch", *extra],
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600,
check=False, env={**os.environ, "RETROBIOS_OFFLINE": "1"},
)
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
self.assertRegex(proc.stderr, r"ERROR: .*nosuch")
class AMissingInputFileIsNamed(unittest.TestCase):
"""The documented `--from-md5-file missing.txt` ended on a
FileNotFoundError at the bottom of a stack trace."""
def _refused(self, argv: list[str], message: str) -> None:
proc = subprocess.run(
[sys.executable, *argv], capture_output=True, text=True,
cwd=str(REPO_ROOT), timeout=600, check=False,
env={**os.environ, "RETROBIOS_OFFLINE": "1"},
)
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
self.assertIn(message, proc.stderr)
def test_a_missing_hash_list_is_named(self):
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
self._refused(
["scripts/generate_pack.py", "--platform", "batocera", "--from-md5-file",
"nosuch.txt", "--offline", "--output-dir", tmp],
"ERROR: --from-md5-file nosuch.txt: No such file or directory",
)
def test_a_missing_data_directory_registry_is_named(self):
for argv in (["scripts/refresh_data_dirs.py", "--registry", "nosuch.yml", "--dry-run"],
["scripts/check_buildbot_system.py", "--registry", "nosuch.yml"]):
with self.subTest(script=argv[0]):
self._refused(argv, "ERROR: Registry not found: nosuch.yml")
class ASystemReportCountsTheClaimsThePackCounts(unittest.TestCase):
"""`--platform retroarch --system panasonic-3do` packed MAME 2009's
taitofx1.zip, which MAME 2016 claims for that system, and the report's
core coverage did not name it: the builder read other_claims, verify
read only the entry's own system."""
def test_a_claim_riding_on_an_entry_selects_it(self):
from packextras import extras_for_systems, report_owned_by_systems
entry = {
"name": "taitofx1.zip", "system": "arcade", "systems": ["arcade"],
"other_claims": [{"name": "taitofx1.zip", "system": None,
"systems": ["arcade", "panasonic-3do"]}],
}
extra = {"source_system": "arcade", "source_systems": ["arcade"],
"other_claims": [{"source_system": None,
"source_systems": ["arcade", "panasonic-3do"]}]}
for system in ("panasonic-3do", "arcade", "sony-playstation"):
with self.subTest(system=system):
self.assertEqual(
bool(report_owned_by_systems([entry], [system])),
bool(extras_for_systems([extra], [system])),
)
self.assertEqual(report_owned_by_systems([entry], ["panasonic-3do"]), [entry])
class VerifyReportsTheRequiredOnlyPack(unittest.TestCase):
"""`generate_pack --emulator mame --required-only` held 164 archives and
the only report, `verify --emulator mame`, credited 2302 paths:
--required-only was refused by verify, and the archive rule existed in
the builder only."""
def setUp(self):
previous = os.getcwd()
os.chdir(REPO_ROOT)
self.addCleanup(os.chdir, previous)
def test_the_report_holds_the_paths_the_pack_holds(self):
import contextlib
import io
from verify import verify_emulator
db = common.load_database("database.json")
for name in ("beetle_psx", "dolphin", "fbneo", "geolith", "mame"):
with self.subTest(emulator=name):
record = builder.PackRecord()
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
builder.generate_emulator_pack(
[name], "emulators", db, "bios", "", offline=True,
platforms_dir="platforms", record=record, required_only=True,
)
report = verify_emulator([name], "emulators", db,
platforms_dir="platforms", required_only=True)
full = verify_emulator([name], "emulators", db, platforms_dir="platforms")
self.assertEqual(sorted(report["pack_destinations"]), sorted(record.files))
self.assertLessEqual(set(report["pack_destinations"]),
set(full["pack_destinations"]))
def test_every_mode_takes_the_flag(self):
for argv in (["--platform", "lakka"], ["--emulator", "handy"],
["--system", "atari-lynx"]):
with self.subTest(argv=argv):
proc = subprocess.run(
[sys.executable, "scripts/verify.py", *argv, "--required-only"],
cwd=REPO_ROOT, capture_output=True, text=True, timeout=600, check=False,
)
self.assertEqual(proc.returncode, 0, proc.stderr[-1500:])
class StrictVerifyFailsOnACriticalMiss(unittest.TestCase):
"""verify exited 0 with a required file missing on an md5 platform, so
the pipeline printed "verify OK" over a pack the frontend rejects."""