mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-22 01:27:52 -06:00
chore: merge engine fixes, third pass
This commit is contained in:
commit
7a98d73d79
37 files changed
+985
-186
No files matched your search
@@ -23,6 +23,7 @@ from common import ( # noqa: E402
|
||||
artifact_lock,
|
||||
hold_artifact_lock,
|
||||
hold_repository_outputs,
|
||||
publish_hold,
|
||||
)
|
||||
|
||||
|
||||
@@ -218,7 +219,7 @@ class PipelineHoldsItsFixedOutputs(unittest.TestCase):
|
||||
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=120,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
||||
self.assertIn("another pipeline run is writing database.json", proc.stderr)
|
||||
self.assertIn("another run is writing database.json", proc.stderr)
|
||||
self.assertNotIn("generate database", proc.stdout.lower())
|
||||
|
||||
def test_two_holds_of_the_repository_exclude_each_other(self):
|
||||
@@ -228,6 +229,59 @@ class PipelineHoldsItsFixedOutputs(unittest.TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class StandaloneWritersHonourTheRepositoryHold(unittest.TestCase):
|
||||
"""Only pipeline runs took the repository's lock: generate_db.py beside
|
||||
a pipeline replaced database.json between its verify and its packs, and
|
||||
a manifest run on install/ failed the pipeline's own manifest step."""
|
||||
|
||||
def _beside_a_pipeline(self, argv: list[str]) -> subprocess.CompletedProcess:
|
||||
env = {k: v for k, v in os.environ.items() if k != "RETROBIOS_HELD_ARTIFACT_DIR"}
|
||||
return subprocess.run(
|
||||
[sys.executable, *argv], capture_output=True, text=True,
|
||||
cwd=str(REPO_ROOT), timeout=300, env=env, check=False,
|
||||
)
|
||||
|
||||
def test_a_writer_of_a_fixed_output_is_refused(self):
|
||||
with hold_repository_outputs(str(REPO_ROOT)):
|
||||
for argv in (
|
||||
["scripts/generate_db.py", "--bios-dir", "bios", "--output", "database.json"],
|
||||
["scripts/generate_pack.py", "--all", "--manifest", "--offline",
|
||||
"--output-dir", "install"],
|
||||
["scripts/generate_games_index.py", "--quiet"],
|
||||
):
|
||||
with self.subTest(script=argv[0]):
|
||||
proc = self._beside_a_pipeline(argv)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout[-500:] + proc.stderr[-500:])
|
||||
self.assertIn("another run is writing database.json", proc.stderr)
|
||||
self.assertNotIn("Traceback", proc.stderr)
|
||||
|
||||
def _child(self, target: str, env: dict) -> subprocess.CompletedProcess:
|
||||
code = (
|
||||
"import sys; sys.path.insert(0, 'scripts')\n"
|
||||
"from common import artifact_lock, repository_output_lock\n"
|
||||
f"with repository_output_lock({target!r}), artifact_lock({self.scratch!r}):\n"
|
||||
" pass\n"
|
||||
)
|
||||
return subprocess.run([sys.executable, "-c", code], cwd=str(REPO_ROOT), env=env,
|
||||
capture_output=True, text=True, timeout=60, check=False)
|
||||
|
||||
def test_a_step_the_pipeline_spawns_passes(self):
|
||||
"""A pipeline holds the root and its --output-dir, and its steps
|
||||
write both: the environment names the two, os.pathsep apart."""
|
||||
self.scratch = _tempdir(self)
|
||||
with hold_repository_outputs(str(REPO_ROOT)), publish_hold(str(REPO_ROOT)), \
|
||||
hold_artifact_lock(self.scratch):
|
||||
proc = self._child(str(REPO_ROOT / "install" / "games"), dict(os.environ))
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
|
||||
def test_a_scratch_output_takes_nothing(self):
|
||||
self.scratch = _tempdir(self)
|
||||
env = {k: v for k, v in os.environ.items() if k != "RETROBIOS_HELD_ARTIFACT_DIR"}
|
||||
with hold_repository_outputs(str(REPO_ROOT)):
|
||||
proc = self._child(os.path.join(self.scratch, "database.json"), env)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
|
||||
|
||||
class HeldLockTest(unittest.TestCase):
|
||||
"""pipeline.py took the lock step by step: between the pack build and the
|
||||
integrity check dist/ was free, another run's purge emptied it, and the
|
||||
|
||||
@@ -296,5 +296,43 @@ class RestoreSkipsDownloadsInProgress(unittest.TestCase):
|
||||
self.assertIn(str(cache / "done.bin"), index.values())
|
||||
|
||||
|
||||
class ATerminatedBuildLeavesNoScratch(unittest.TestCase):
|
||||
"""`timeout 120 generate_pack.py --platform RetroArch` left a 3.4 GB
|
||||
hidden .partial in dist/: SIGTERM ended the process without running the
|
||||
cleanup a failed write runs."""
|
||||
|
||||
CHILD = (
|
||||
"import sys, time\n"
|
||||
"sys.path.insert(0, 'scripts')\n"
|
||||
"from common import run_cli\n"
|
||||
"from generate_pack import _writing_pack\n"
|
||||
"def main():\n"
|
||||
" with _writing_pack(sys.argv[1]) as zf:\n"
|
||||
" zf.writestr('a.bin', b'x')\n"
|
||||
" print('writing', flush=True)\n"
|
||||
" time.sleep(60)\n"
|
||||
"run_cli(main)\n"
|
||||
)
|
||||
|
||||
def test_sigterm_removes_the_partial_pack(self):
|
||||
import signal # noqa: PLC0415
|
||||
import subprocess # noqa: PLC0415
|
||||
|
||||
if not hasattr(signal, "SIGTERM") or os.name != "posix":
|
||||
self.skipTest("POSIX signals")
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
|
||||
pack = Path(tmp) / "Demo_BIOS_Pack.zip"
|
||||
child = subprocess.Popen(
|
||||
[sys.executable, "-c", self.CHILD, str(pack)], cwd=REPO_ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
)
|
||||
self.assertEqual(child.stdout.readline().strip(), "writing")
|
||||
self.assertTrue(list(Path(tmp).glob(".*.partial")))
|
||||
child.send_signal(signal.SIGTERM)
|
||||
child.communicate(timeout=60)
|
||||
self.assertEqual(child.returncode, 128 + signal.SIGTERM)
|
||||
self.assertEqual(sorted(p.name for p in Path(tmp).iterdir()), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+105
-17
@@ -118,6 +118,24 @@ class ReadmeRegressions(unittest.TestCase):
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertIn("needs Python 3.9", proc.stderr)
|
||||
|
||||
def test_the_readme_build_command_names_the_floor(self):
|
||||
"""The README's first command is generate_pack.py. Under 3.8 it
|
||||
stopped on a raw SyntaxError at a parenthesized `with`."""
|
||||
import shutil # noqa: PLC0415
|
||||
|
||||
python38 = shutil.which("python3.8")
|
||||
if python38 is None:
|
||||
if os.environ.get("CI"):
|
||||
self.fail("CI must provide python3.8")
|
||||
self.skipTest("no python3.8 on PATH")
|
||||
proc = subprocess.run(
|
||||
[python38, "-I", "scripts/generate_pack.py", "--help"],
|
||||
cwd=ROOT, capture_output=True, text=True, timeout=60, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertNotIn("SyntaxError", proc.stderr)
|
||||
self.assertIn("generate_pack.py needs Python 3.9", proc.stderr)
|
||||
|
||||
def test_the_floor_interpreter_imports_every_script(self):
|
||||
"""Every module under scripts/ is loaded by a real interpreter of the
|
||||
floor version, as the installer's 3.8 floor is proven by running it.
|
||||
@@ -948,6 +966,20 @@ class CatalogRatioRegressions(unittest.TestCase):
|
||||
|
||||
|
||||
class PipelineRegressions(unittest.TestCase):
|
||||
def test_integrity_checks_the_variants_step_4_built(self):
|
||||
"""Step 6 ran --verify-packs without --all-variants: the five
|
||||
narrowed packs were never opened and an empty _Platform passed."""
|
||||
import argparse # noqa: PLC0415
|
||||
|
||||
def command(**flags) -> list[str]:
|
||||
base = {"output_dir": "dist", "include_archived": False,
|
||||
"target": None, "all_variants": False}
|
||||
return pipeline.integrity_command(argparse.Namespace(**{**base, **flags}))
|
||||
|
||||
self.assertIn("--all-variants", command(all_variants=True))
|
||||
self.assertNotIn("--all-variants", command())
|
||||
self.assertEqual(command(target="switch")[-2:], ["--target", "switch"])
|
||||
|
||||
def test_pack_parser_removes_source_metadata(self):
|
||||
output = "\n".join(
|
||||
[
|
||||
@@ -2734,6 +2766,10 @@ class TheProjectInstallsWithPip(unittest.TestCase):
|
||||
import importlib.util # noqa: PLC0415
|
||||
|
||||
if importlib.util.find_spec("setuptools") is None:
|
||||
# The dev extra and CI install it: skipped there, the guard held
|
||||
# nothing on any run.
|
||||
if os.environ.get("CI"):
|
||||
self.fail("CI must install setuptools")
|
||||
self.skipTest("setuptools is not installed")
|
||||
program = (
|
||||
"from setuptools.dist import Distribution\n"
|
||||
@@ -2808,27 +2844,35 @@ class ACheckThatCannotAnswerDoesNotPass(unittest.TestCase):
|
||||
self.assertEqual(unsatisfied, ["bios/absent.bin"])
|
||||
|
||||
def test_an_unreachable_buildbot_is_not_a_fresh_verdict(self):
|
||||
source = (ROOT / "scripts" / "check_buildbot_system.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn(
|
||||
'if report.get("error") or pending(report):',
|
||||
source.split("def main(")[-1],
|
||||
"main() ignores the error the report carries",
|
||||
)
|
||||
"""Every HEAD unanswered printed "16 unknown" and exited 0, so the
|
||||
online pipeline printed check_buildbot OK."""
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import check_buildbot_system as cbs # noqa: PLC0415
|
||||
|
||||
registry = str(ROOT / "platforms" / "_data_dirs.yml")
|
||||
tracked = cbs.load_tracked_entries(registry)
|
||||
self.assertTrue(tracked)
|
||||
cases = {
|
||||
"index unreachable": mock.patch.object(
|
||||
cbs, "fetch_index", side_effect=OSError("down")),
|
||||
"every HEAD unanswered": mock.patch.object(
|
||||
cbs, "fetch_index", return_value=set(tracked)),
|
||||
}
|
||||
for label, index in cases.items():
|
||||
with self.subTest(label), index, \
|
||||
mock.patch.object(cbs, "get_remote_etag", return_value=None), \
|
||||
mock.patch.object(sys, "argv", ["check_buildbot_system.py",
|
||||
"--registry", registry]), \
|
||||
contextlib.redirect_stdout(io.StringIO()), \
|
||||
self.assertRaises(SystemExit) as stop:
|
||||
cbs.main()
|
||||
self.assertEqual(stop.exception.code, 1)
|
||||
|
||||
def test_a_missing_profile_directory_says_so(self):
|
||||
import io as _io
|
||||
|
||||
from scripts import common as _common
|
||||
|
||||
stderr = _io.StringIO()
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
profiles = _common.load_emulator_profiles(
|
||||
str(ROOT / "no-such-emulator-dir")
|
||||
)
|
||||
self.assertEqual(profiles, {})
|
||||
self.assertIn("no emulator profile directory", stderr.getvalue())
|
||||
with self.assertRaisesRegex(_common.InputMissing, "no emulator profile directory"):
|
||||
_common.load_emulator_profiles(str(ROOT / "no-such-emulator-dir"))
|
||||
|
||||
|
||||
class TestEntryPointsRunEveryClass(unittest.TestCase):
|
||||
@@ -4145,6 +4189,8 @@ class EveryToolRefusesAFlagItWouldDrop(unittest.TestCase):
|
||||
["--offline", "--all-variants", "--source", "platform"],
|
||||
["--offline", "--all-variants", "--source", "full"],
|
||||
["--offline", "--skip-packs", "--source", "full"],
|
||||
["--offline", "--source", "truth", "--full-narrowings"],
|
||||
["--offline", "--source", "platform", "--full-narrowings"],
|
||||
],
|
||||
}
|
||||
|
||||
@@ -7138,5 +7184,47 @@ class OpenMsxSystemRomsAreFoundBySha1(unittest.TestCase):
|
||||
self.assertEqual(named, [])
|
||||
|
||||
|
||||
class TupleAnnotationsMatchTheirReturns(unittest.TestCase):
|
||||
"""`_by_affinity.score` gained a fifth sort key while its annotation kept
|
||||
four, so the signature misread the order that sends C-Dogs'
|
||||
gamecontrollerdb.txt behind a neutral copy."""
|
||||
|
||||
@staticmethod
|
||||
def _own_returns(fn: ast.AST):
|
||||
stack = list(ast.iter_child_nodes(fn))
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
||||
continue
|
||||
if isinstance(node, ast.Return):
|
||||
yield node
|
||||
stack.extend(ast.iter_child_nodes(node))
|
||||
|
||||
def test_a_fixed_tuple_annotation_counts_its_returned_values(self):
|
||||
sources = sorted((ROOT / "scripts").rglob("*.py")) + [ROOT / "install.py"]
|
||||
mismatched = []
|
||||
for path in sources:
|
||||
for fn in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
|
||||
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
hint = fn.returns
|
||||
if not (isinstance(hint, ast.Subscript) and isinstance(hint.value, ast.Name)
|
||||
and hint.value.id == "tuple"):
|
||||
continue
|
||||
declared = hint.slice.elts if isinstance(hint.slice, ast.Tuple) else [hint.slice]
|
||||
if any(isinstance(e, ast.Constant) and e.value is Ellipsis for e in declared):
|
||||
continue
|
||||
for ret in self._own_returns(fn):
|
||||
value = ret.value
|
||||
if (isinstance(value, ast.Tuple)
|
||||
and not any(isinstance(e, ast.Starred) for e in value.elts)
|
||||
and len(value.elts) != len(declared)):
|
||||
mismatched.append(
|
||||
f"{path.relative_to(ROOT)}:{ret.lineno} "
|
||||
f"{len(value.elts)} returned, {len(declared)} declared"
|
||||
)
|
||||
self.assertEqual(mismatched, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -165,5 +165,30 @@ class AmberelecCarriesTheMachines(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TheCountLineKeepsTreesOutOfTheBaseline(unittest.TestCase):
|
||||
"""MyMinUI declares no file, and its pack line read "6944 baseline": the
|
||||
fbneo trees its cores read were counted as the platform's list."""
|
||||
|
||||
def test_a_platform_declaring_nothing_has_no_baseline(self) -> None:
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
previous = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
self.addCleanup(os.chdir, previous)
|
||||
db = common.load_database("database.json")
|
||||
record = builder.PackRecord()
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
builder.generate_pack(
|
||||
"myminui", "platforms", db, "bios", "tmp", offline=True, record=record,
|
||||
data_registry=common.load_data_dir_registry("platforms"),
|
||||
)
|
||||
line = next(l for l in out.getvalue().splitlines() if "files packed" in l)
|
||||
self.assertIn("(0 baseline + ", line)
|
||||
if record.trees:
|
||||
self.assertIn(f"+ {len(record.trees)} in core data trees + ", line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -95,6 +95,38 @@ class CoverageCache(unittest.TestCase):
|
||||
self._run()
|
||||
self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1)
|
||||
|
||||
def test_a_purge_racing_another_generator_does_not_fail(self):
|
||||
"""generate_site and generate_readme purge the same directory: one
|
||||
unlinking a pickle the other had listed ended on FileNotFoundError."""
|
||||
cache_dir = self.work / "cache"
|
||||
cache_dir.mkdir()
|
||||
ghost = cache_dir / "gone.pickle"
|
||||
real_glob = Path.glob
|
||||
|
||||
def listing_a_vanished_file(path, pattern):
|
||||
found = list(real_glob(path, pattern))
|
||||
return [*found, ghost] if path == cache_dir else found
|
||||
|
||||
with mock.patch.object(Path, "glob", listing_a_vanished_file):
|
||||
coverages = self._run()
|
||||
self.assertEqual(sorted(coverages), ["alpha", "beta"])
|
||||
self.assertEqual(len(list(cache_dir.glob("*.pickle"))), 1)
|
||||
|
||||
def test_a_missing_input_directory_stops_the_run(self):
|
||||
"""A missing --emulators-dir skipped every platform as if it had no
|
||||
YAML, then replaced the cache with the empty result."""
|
||||
self._run()
|
||||
kept = sorted((self.work / "cache").glob("*.pickle"))
|
||||
|
||||
def missing(*_args, **_kwargs):
|
||||
raise generate_readme.InputMissing("no emulator profile directory at nosuch")
|
||||
|
||||
with mock.patch.object(generate_readme, "compute_coverage", missing), \
|
||||
mock.patch.object(generate_readme, "_coverage_inputs", return_value="other"), \
|
||||
self.assertRaises(generate_readme.InputMissing):
|
||||
self._run()
|
||||
self.assertEqual(sorted((self.work / "cache").glob("*.pickle")), kept)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -31,5 +31,35 @@ class FinalizeByExactName(unittest.TestCase):
|
||||
self.assertTrue(ok)
|
||||
|
||||
|
||||
class APackFromAnotherRunIsLeftAlone(unittest.TestCase):
|
||||
"""A one-platform build verified, recompressed and gave a new manifest to
|
||||
every pack in the directory: MiSTer's lost its pack_systems fields after a
|
||||
MinUI run, and a bizhawk run rewrote 52 GB of packs it never built."""
|
||||
|
||||
def test_a_second_platform_does_not_rewrite_the_first(self):
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
|
||||
def build(platform: str) -> str:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/generate_pack.py", "--platform", platform,
|
||||
"--offline", "--output-dir", tmp],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, timeout=600, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout[-2000:] + proc.stderr[-2000:])
|
||||
return proc.stdout
|
||||
|
||||
build("misterfpga")
|
||||
(first,) = Path(tmp).glob("MiSTer*_BIOS_Pack.zip")
|
||||
before = first.read_bytes()
|
||||
output = build("minui")
|
||||
self.assertNotIn(first.name, output)
|
||||
self.assertEqual(first.read_bytes(), before)
|
||||
with zipfile.ZipFile(first) as zf:
|
||||
manifest = json.loads(zf.read("manifest.json"))
|
||||
self.assertIn("install_list", manifest)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -182,6 +182,28 @@ class EveryNarrowingIsCompared(unittest.TestCase):
|
||||
"compare the new narrowing in check_narrowed_packs, "
|
||||
"or write in NARROWINGS_NOT_COMPARED why it is not")
|
||||
|
||||
def test_each_emulator_pack_narrowing_is_compared(self):
|
||||
"""check_emulator_packs built every pack without --region or
|
||||
--required-only, while the table above counted both as compared:
|
||||
it describes the platform packs only."""
|
||||
import inspect
|
||||
|
||||
import pipeline
|
||||
from generate_pack import _narrowings, generate_emulator_pack
|
||||
|
||||
taken = (set(inspect.signature(_narrowings).parameters)
|
||||
& set(inspect.signature(generate_emulator_pack).parameters))
|
||||
self.assertEqual(taken, set(pipeline.EMULATOR_NARROWINGS_COMPARED))
|
||||
profile = {"files": [{"name": "a.bin", "region": "japan"},
|
||||
{"name": "b.bin", "region": ["north-america", "europe"]}]}
|
||||
used = {key for _label, narrowing in pipeline.emulator_variants([profile], False)
|
||||
for key in narrowing}
|
||||
# standalone is the loop around the variants, both builds of each profile.
|
||||
self.assertEqual(used, pipeline.EMULATOR_NARROWINGS_COMPARED - {"standalone"})
|
||||
regions = [n["regions"] for _l, n in pipeline.emulator_variants([profile], True)
|
||||
if "regions" in n]
|
||||
self.assertEqual(len(regions), 3)
|
||||
|
||||
def test_the_variants_reach_every_compared_dimension(self):
|
||||
import pipeline
|
||||
from common import load_emulator_profiles, load_platform_config
|
||||
@@ -246,7 +268,7 @@ class ThePipelineComparesPaths(unittest.TestCase):
|
||||
def test_the_run_compares_every_emulator_pack(self):
|
||||
source = (REPO_ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
|
||||
self.assertIn("ok = check_pack_destinations(destinations_file, pack_output) and ok", source)
|
||||
self.assertIn("ok = check_emulator_packs() and ok", source)
|
||||
self.assertIn("ok = check_emulator_packs(full=args.full_narrowings) and ok", source)
|
||||
self.assertIn('"--destinations", destinations_file', source)
|
||||
|
||||
|
||||
|
||||
@@ -93,6 +93,31 @@ class RefreshConcurrency(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(seen, [True])
|
||||
|
||||
def test_a_current_tree_is_not_waited_for(self):
|
||||
"""A pipeline reads every tree for its whole run. A build beside it
|
||||
waited on the exclusive lock before asking the remote, which would
|
||||
have answered that nothing changed."""
|
||||
import fcntl # noqa: PLC0415
|
||||
|
||||
self.cache.mkdir(parents=True)
|
||||
rdd._record_version("pak", {"sha": "e1"}, self.versions)
|
||||
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
|
||||
"local_cache": str(self.cache)}
|
||||
result: list = []
|
||||
with open(self.cache.with_name(".pak.lock"), "a") as reader, \
|
||||
mock.patch.object(rdd, "_get_remote_etag", return_value="e1"):
|
||||
fcntl.flock(reader, fcntl.LOCK_SH)
|
||||
worker = threading.Thread(target=lambda: result.append(
|
||||
rdd.refresh_entry("pak", entry, versions_path=self.versions)
|
||||
), daemon=True)
|
||||
worker.start()
|
||||
worker.join(timeout=20)
|
||||
finished = not worker.is_alive()
|
||||
fcntl.flock(reader, fcntl.LOCK_UN)
|
||||
worker.join(timeout=20)
|
||||
self.assertTrue(finished, "the refresh waited for a reader")
|
||||
self.assertEqual(result, [False])
|
||||
|
||||
def test_version_file_is_never_left_truncated(self):
|
||||
Path(self.versions).parent.mkdir(parents=True)
|
||||
Path(self.versions).write_text(json.dumps({"keep": {"sha": "1"}}))
|
||||
|
||||
@@ -262,6 +262,219 @@ class AMissingBiosDirIsRefused(unittest.TestCase):
|
||||
self.assertFalse(Path(out).exists())
|
||||
|
||||
|
||||
def _scripts_taking(flag: str) -> list[Path]:
|
||||
pattern = re.compile(r'add_argument\(\s*"%s"' % re.escape(flag))
|
||||
return [
|
||||
path for path in sorted((REPO_ROOT / "scripts").glob("*.py"))
|
||||
+ sorted((REPO_ROOT / "scripts" / "scraper").glob("*.py"))
|
||||
if pattern.search(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
|
||||
|
||||
def _command(path: Path) -> list[str]:
|
||||
if path.parent.name == "scraper":
|
||||
return [sys.executable, "-m", f"scripts.scraper.{path.stem}"]
|
||||
return [sys.executable, f"scripts/{path.name}"]
|
||||
|
||||
|
||||
class AMissingProfileDirIsRefused(unittest.TestCase):
|
||||
"""A mistyped --emulators-dir read as "no emulator": generate_pack built
|
||||
a pack without its core files marked OK, verify --strict passed, and
|
||||
sitesources wrote {} over source-paths.json, every one exiting 0."""
|
||||
|
||||
EXTRA = {
|
||||
"arcade_parent_importer": ["--all", "--offline"],
|
||||
"fileless_audit": ["--all"],
|
||||
"generate_pack": ["--platform", "retroarch", "--system", "atari-lynx",
|
||||
"--offline", "--output-dir", "{out}"],
|
||||
"generate_site": ["--docs-dir", "{out}"],
|
||||
"generate_truth": ["--all", "--output-dir", "{out}"],
|
||||
"mame_ref_audit": ["mame"],
|
||||
"profile_sync": ["--all", "--triage"],
|
||||
"romset_dat_importer": ["--source", "mame", "--pack", "{dat}", "--output", "{out}/r.json"],
|
||||
"sitesources": ["--cache-dir", "{cache}", "--output", "{out}/source-paths.json"],
|
||||
"validate_pr": ["bios/Sony/PlayStation/scph5501.bin"],
|
||||
"verify": ["--platform", "retroarch", "--strict"],
|
||||
}
|
||||
|
||||
def test_every_script_taking_an_emulators_dir_refuses_a_missing_one(self):
|
||||
scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
|
||||
self.addCleanup(scratch.cleanup)
|
||||
root = Path(scratch.name)
|
||||
(root / "cache").mkdir()
|
||||
(root / "empty.dat").write_bytes(b"")
|
||||
values = {"out": str(root / "out"), "cache": str(root / "cache"),
|
||||
"dat": str(root / "empty.dat")}
|
||||
taking = _scripts_taking("--emulators-dir")
|
||||
self.assertGreater(len(taking), 15)
|
||||
tracked = ["git", "diff", "--name-only"]
|
||||
before = subprocess.run(tracked, cwd=REPO_ROOT, capture_output=True, text=True).stdout
|
||||
for path in taking:
|
||||
extra = [arg.format(**values) for arg in self.EXTRA.get(path.stem, [])]
|
||||
with self.subTest(script=path.stem):
|
||||
proc = subprocess.run(
|
||||
[*_command(path), "--emulators-dir", "nosuch", *extra],
|
||||
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600,
|
||||
check=False, env={**os.environ, "RETROBIOS_OFFLINE": "1"},
|
||||
)
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
|
||||
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
|
||||
self.assertRegex(proc.stderr, r"ERROR: .*nosuch")
|
||||
written = [
|
||||
str(p.relative_to(root)) for p in (root / "out").rglob("*")
|
||||
if p.is_file() and p.name != ".lock"
|
||||
]
|
||||
self.assertEqual(written, [])
|
||||
# Read as empty, generate_site rewrote published-parts.json and the
|
||||
# FAQ, and sitesources source-paths.json.
|
||||
after = subprocess.run(tracked, cwd=REPO_ROOT, capture_output=True, text=True).stdout
|
||||
self.assertEqual(after, before)
|
||||
|
||||
|
||||
class AMissingPlatformsDirIsRefused(unittest.TestCase):
|
||||
"""`verify.py --platforms-dir noplat` ended on a FileNotFoundError at the
|
||||
bottom of a stack trace, and validate_pr read it as no platform at all
|
||||
and exited 0."""
|
||||
|
||||
EXTRA = {
|
||||
"auto_fetch": ["--all", "--dry-run"],
|
||||
"check_release": ["--tag", "v1"],
|
||||
"diff_truth": ["--all"],
|
||||
"export_native": ["--all", "--output-dir", "{out}"],
|
||||
"generate_pack": ["--platform", "retroarch", "--offline", "--output-dir", "{out}"],
|
||||
"generate_site": ["--docs-dir", "{out}"],
|
||||
"generate_truth": ["--all", "--output-dir", "{out}"],
|
||||
"release_notes": ["--whats-new", "{notes}", "--output", "{out}/notes.md"],
|
||||
"release_record": ["{out}", "--tag", "v1"],
|
||||
"romset_dat_importer": ["--source", "mame", "--pack", "{dat}", "--output", "{out}/r.json"],
|
||||
"validate_pr": ["bios/Sony/PlayStation/scph5501.bin"],
|
||||
"validate_site": ["--site-dir", "{out}"],
|
||||
"verify": ["--platform", "retroarch"],
|
||||
}
|
||||
|
||||
def test_every_script_taking_a_platforms_dir_refuses_a_missing_one(self):
|
||||
scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
|
||||
self.addCleanup(scratch.cleanup)
|
||||
root = Path(scratch.name)
|
||||
(root / "empty.dat").write_bytes(b"")
|
||||
(root / "notes.md").write_text("Notes.\n")
|
||||
values = {"out": str(root / "out"), "dat": str(root / "empty.dat"),
|
||||
"notes": str(root / "notes.md")}
|
||||
taking = _scripts_taking("--platforms-dir")
|
||||
self.assertGreater(len(taking), 15)
|
||||
for path in taking:
|
||||
extra = [arg.format(**values) for arg in self.EXTRA.get(path.stem, [])]
|
||||
with self.subTest(script=path.stem):
|
||||
proc = subprocess.run(
|
||||
[*_command(path), "--platforms-dir", "nosuch", *extra],
|
||||
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600,
|
||||
check=False, env={**os.environ, "RETROBIOS_OFFLINE": "1"},
|
||||
)
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
|
||||
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
|
||||
self.assertRegex(proc.stderr, r"ERROR: .*nosuch")
|
||||
|
||||
|
||||
class AMissingInputFileIsNamed(unittest.TestCase):
|
||||
"""The documented `--from-md5-file missing.txt` ended on a
|
||||
FileNotFoundError at the bottom of a stack trace."""
|
||||
|
||||
def _refused(self, argv: list[str], message: str) -> None:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, *argv], capture_output=True, text=True,
|
||||
cwd=str(REPO_ROOT), timeout=600, check=False,
|
||||
env={**os.environ, "RETROBIOS_OFFLINE": "1"},
|
||||
)
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout[-500:])
|
||||
self.assertNotIn("Traceback", proc.stderr, proc.stderr[-1500:])
|
||||
self.assertIn(message, proc.stderr)
|
||||
|
||||
def test_a_missing_hash_list_is_named(self):
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
|
||||
self._refused(
|
||||
["scripts/generate_pack.py", "--platform", "batocera", "--from-md5-file",
|
||||
"nosuch.txt", "--offline", "--output-dir", tmp],
|
||||
"ERROR: --from-md5-file nosuch.txt: No such file or directory",
|
||||
)
|
||||
|
||||
def test_a_missing_data_directory_registry_is_named(self):
|
||||
for argv in (["scripts/refresh_data_dirs.py", "--registry", "nosuch.yml", "--dry-run"],
|
||||
["scripts/check_buildbot_system.py", "--registry", "nosuch.yml"]):
|
||||
with self.subTest(script=argv[0]):
|
||||
self._refused(argv, "ERROR: Registry not found: nosuch.yml")
|
||||
|
||||
|
||||
class ASystemReportCountsTheClaimsThePackCounts(unittest.TestCase):
|
||||
"""`--platform retroarch --system panasonic-3do` packed MAME 2009's
|
||||
taitofx1.zip, which MAME 2016 claims for that system, and the report's
|
||||
core coverage did not name it: the builder read other_claims, verify
|
||||
read only the entry's own system."""
|
||||
|
||||
def test_a_claim_riding_on_an_entry_selects_it(self):
|
||||
from packextras import extras_for_systems, report_owned_by_systems
|
||||
|
||||
entry = {
|
||||
"name": "taitofx1.zip", "system": "arcade", "systems": ["arcade"],
|
||||
"other_claims": [{"name": "taitofx1.zip", "system": None,
|
||||
"systems": ["arcade", "panasonic-3do"]}],
|
||||
}
|
||||
extra = {"source_system": "arcade", "source_systems": ["arcade"],
|
||||
"other_claims": [{"source_system": None,
|
||||
"source_systems": ["arcade", "panasonic-3do"]}]}
|
||||
for system in ("panasonic-3do", "arcade", "sony-playstation"):
|
||||
with self.subTest(system=system):
|
||||
self.assertEqual(
|
||||
bool(report_owned_by_systems([entry], [system])),
|
||||
bool(extras_for_systems([extra], [system])),
|
||||
)
|
||||
self.assertEqual(report_owned_by_systems([entry], ["panasonic-3do"]), [entry])
|
||||
|
||||
|
||||
class VerifyReportsTheRequiredOnlyPack(unittest.TestCase):
|
||||
"""`generate_pack --emulator mame --required-only` held 164 archives and
|
||||
the only report, `verify --emulator mame`, credited 2302 paths:
|
||||
--required-only was refused by verify, and the archive rule existed in
|
||||
the builder only."""
|
||||
|
||||
def setUp(self):
|
||||
previous = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
self.addCleanup(os.chdir, previous)
|
||||
|
||||
def test_the_report_holds_the_paths_the_pack_holds(self):
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
from verify import verify_emulator
|
||||
|
||||
db = common.load_database("database.json")
|
||||
for name in ("beetle_psx", "dolphin", "fbneo", "geolith", "mame"):
|
||||
with self.subTest(emulator=name):
|
||||
record = builder.PackRecord()
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()):
|
||||
builder.generate_emulator_pack(
|
||||
[name], "emulators", db, "bios", "", offline=True,
|
||||
platforms_dir="platforms", record=record, required_only=True,
|
||||
)
|
||||
report = verify_emulator([name], "emulators", db,
|
||||
platforms_dir="platforms", required_only=True)
|
||||
full = verify_emulator([name], "emulators", db, platforms_dir="platforms")
|
||||
self.assertEqual(sorted(report["pack_destinations"]), sorted(record.files))
|
||||
self.assertLessEqual(set(report["pack_destinations"]),
|
||||
set(full["pack_destinations"]))
|
||||
|
||||
def test_every_mode_takes_the_flag(self):
|
||||
for argv in (["--platform", "lakka"], ["--emulator", "handy"],
|
||||
["--system", "atari-lynx"]):
|
||||
with self.subTest(argv=argv):
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/verify.py", *argv, "--required-only"],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, timeout=600, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr[-1500:])
|
||||
|
||||
|
||||
class StrictVerifyFailsOnACriticalMiss(unittest.TestCase):
|
||||
"""verify exited 0 with a required file missing on an md5 platform, so
|
||||
the pipeline printed "verify OK" over a pack the frontend rejects."""
|
||||
|
||||
Reference in new issue
Block a user