"""Regression tests for the holistic reliability and security audit."""
from __future__ import annotations
import ast
import contextlib
import hashlib
import io
import json
import os
import re
import stat
import subprocess
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest import mock
import yaml
ROOT = Path(__file__).resolve().parent.parent
TMP_ROOT = ROOT / "tmp" / "tests"
TMP_ROOT.mkdir(parents=True, exist_ok=True)
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "scripts"))
import install
from scripts import pipeline, region, region_audit
from scripts.common import resolve_local_file, safe_extract_zip
from scripts.generate_pack import (
_emulator_region_group,
generate_target_manifests,
verify_pack_against_platform,
)
class ReadmeRegressions(unittest.TestCase):
"""The README advertises commands; these must stay executable.
Wording is the maintainer's, so nothing here asserts prose. What is
asserted is that every flag and URL the README hands a reader is one the
shipped scripts actually accept.
"""
def _quick_install(self) -> str:
readme = (ROOT / "README.md").read_text(encoding="utf-8")
return readme.split("## Quick Install", 1)[1].split(
"## Download BIOS packs", 1
)[0]
def test_advertised_bootstrap_urls_are_the_shipped_ones(self):
quick_install = self._quick_install()
for script in ("install.sh", "install.ps1"):
url = f"https://raw.githubusercontent.com/Abdess/retrobios/main/{script}"
self.assertIn(url, quick_install, f"{script} bootstrap URL missing")
self.assertIn(
"https://raw.githubusercontent.com/Abdess/retrobios/main/install.py",
(ROOT / "install.sh").read_text(encoding="utf-8"),
"install.sh must fetch install.py from the ref the README advertises",
)
def test_advertised_installer_flags_exist(self):
quick_install = self._quick_install()
advertised = set(re.findall(r"(? 40, "key material looks truncated")
def test_the_release_signs_the_list_and_ships_the_signature(self):
process = (ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8")
self.assertIn(
"ssh-keygen -Y sign", process, "the release no longer signs the list"
)
upload = next(
line for line in process.splitlines()
if line.startswith("for f in dist/SHA256SUMS.txt")
)
self.assertIn(
"dist/SHA256SUMS.txt.sig", upload,
"the signature is produced but never uploaded",
)
def test_the_documented_verification_matches_the_published_principal(self):
process = (ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8")
self.assertIn("ssh-keygen -Y verify", process)
self.assertIn(f"-I {self.PRINCIPAL}", process)
self.assertIn("allowed_signers", process)
class WorkflowRegressions(unittest.TestCase):
"""The test suite must sit on every road into main.
run-tests lived in a pull_request-only workflow while the work landed by
direct push, so 1,318 cases guarded a road almost nothing took. What is
asserted here is reachability, not the workflow's shape.
"""
@staticmethod
def _workflow(name: str) -> dict:
with (ROOT / ".github" / "workflows" / name).open(encoding="utf-8") as f:
document = yaml.safe_load(f)
# PyYAML resolves the bare `on:` key to the boolean True.
document["on"] = document.pop(True, document.get("on"))
return document
def test_no_validation_step_discards_its_exit_code(self):
"""A check whose result is thrown away is not a check.
The BIOS validation step ended in `|| true`, so validate_pr.py could
exit 1 on a file that failed its hash and the job stayed green. The
report still has to reach the pull request, so the code is recorded
and acted on afterwards rather than swallowed. A best-effort side
action such as adding a label is not a check and keeps its `|| true`.
"""
checks = ("python scripts/", "unittest", "mkdocs build")
for name in ("validate.yml", "deploy-site.yml"):
workflow = self._workflow(name)
for job_name, job in workflow["jobs"].items():
for step in job.get("steps", []):
# A shell continuation puts the command and its `|| true`
# on different lines, so they are rejoined before scanning.
body = str(step.get("run", "")).replace("\\\n", " ")
for line in body.splitlines():
if not any(marker in line for marker in checks):
continue
self.assertNotIn(
"|| true",
line,
f"{name}:{job_name}:{step.get('name', '?')} runs a "
f"check and discards its exit code: {line.strip()}",
)
workflow = self._workflow("validate.yml")
steps = workflow["jobs"]["validate-bios"]["steps"]
gate = [s for s in steps if "rc != " in str(s.get("if", ""))]
self.assertTrue(
gate,
"nothing in validate-bios acts on the validation exit code",
)
def test_the_suite_runs_on_a_direct_push_to_main(self):
workflow = self._workflow("validate.yml")
triggers = workflow["on"]
self.assertIn("push", triggers, "validate.yml no longer runs on push")
self.assertIn("main", triggers["push"]["branches"])
job = next(
(
name
for name, body in workflow["jobs"].items()
if any(
"unittest discover" in str(step.get("run", ""))
for step in body.get("steps", [])
)
),
None,
)
self.assertIsNotNone(job, "no job runs the test suite")
self.assertIsNone(
workflow["jobs"][job].get("if"),
f"{job} carries a condition; the suite must run on push and on PR",
)
def test_both_events_watch_the_same_paths(self):
triggers = self._workflow("validate.yml")["on"]
self.assertEqual(
triggers["pull_request"]["paths"],
triggers["push"]["paths"],
"the two path lists have drifted; a push would skip what a PR checks",
)
def test_pull_request_only_jobs_are_guarded(self):
workflow = self._workflow("validate.yml")
for name, body in workflow["jobs"].items():
uses_pr_context = "github.event.pull_request" in yaml.dump(body)
if uses_pr_context:
self.assertEqual(
body.get("if"),
"github.event_name == 'pull_request'",
f"{name} reads pull request context and would run on a push",
)
def test_no_workflow_relies_on_a_yaml_anchor(self):
"""The workflow parser reads no anchor; PyYAML would hide the break."""
for path in sorted((ROOT / ".github" / "workflows").glob("*.yml")):
for number, line in enumerate(
path.read_text(encoding="utf-8").splitlines(), 1
):
self.assertIsNone(
re.search(r"(?:^|\s)[&*][A-Za-z_]", line),
f"{path.name}:{number} uses a YAML anchor: {line.strip()}",
)
class UnreachableCitationRegressions(unittest.TestCase):
"""A citation must name a place a declared repository can hold.
kenji-nx carried tmp/es-de/ANDROID.md:470-474, a path from the machine
of whoever profiled it. No revision of any declared repository holds it,
so profile_sync could only report it missing, every pass, forever. The
check is offline and runs on every push.
"""
def _hits(self, text: str) -> bool:
import validate_schemas
return bool(validate_schemas._UNREACHABLE_REF.search(" " + text))
def test_a_scratch_directory_is_rejected(self):
self.assertTrue(self._hits("tmp/es-de/ANDROID.md:470-474"))
def test_an_absolute_path_is_rejected(self):
self.assertTrue(self._hits("/home/someone/src/a.c:12"))
self.assertTrue(self._hits(r"C:\work\src\a.c:12"))
def test_a_path_climbing_out_of_the_tree_is_rejected(self):
self.assertTrue(self._hits("../outside/src/a.c:12"))
def test_ordinary_citations_pass(self):
for good in (
"src/core/main.cpp:210",
"libretro.c:5293-5337",
"PCE.emu/src/main/Main.cc:80-91",
"es-de ANDROID.md:470-474",
"see tmp files for details",
):
with self.subTest(citation=good):
self.assertFalse(self._hits(good))
def test_the_corpus_carries_none(self):
import validate_schemas
errors = []
for path in sorted((ROOT / "emulators").glob("*.yml")):
with path.open(encoding="utf-8") as handle:
errors.extend(
validate_schemas._unreachable_citations(path, yaml.safe_load(handle))
)
self.assertEqual(errors, [])
class FaqRegressions(unittest.TestCase):
"""The FAQ states facts the code owns; these tie it back to the source.
Three of its claims had drifted: a pinned MAME version, Adler-32
attributed to Dolphin's IPL instead of its DSP ROMs, and a verbose
per-emulator report described as the only way to catch a bad file on an
existence platform. Readers act on all three.
"""
_MAME_GENERATIONS = {
"MAME 2000": "mame2000",
"MAME 2003": "mame2003",
"MAME 2009": "mame2009",
"MAME 2010": "mame2010",
"MAME 2015": "mame2015",
"MAME 2016": "mame2016",
"current MAME": "mame",
}
@staticmethod
def _faq() -> str:
return (ROOT / "wiki" / "faq.md").read_text(encoding="utf-8")
@staticmethod
def _profile(key: str) -> dict:
with (ROOT / "emulators" / f"{key}.yml").open(encoding="utf-8") as handle:
return yaml.safe_load(handle)
def test_mame_versions_match_their_profiles(self):
# The list is wrapped, so a label and its version can straddle a line.
faq = " ".join(self._faq().split())
for label, key in self._MAME_GENERATIONS.items():
match = re.search(rf"{re.escape(label)} \(([^)]+)\)", faq)
self.assertIsNotNone(match, f"FAQ no longer states a version for {label}")
self.assertEqual(
match.group(1),
self._profile(key)["core_version"],
f"FAQ version for {label} has drifted from emulators/{key}.yml",
)
def test_adler32_is_attributed_to_the_files_that_carry_it(self):
dolphin = self._profile("dolphin")
carriers = {
f["name"] for f in dolphin["files"] if f.get("known_hash_adler32")
}
self.assertTrue(carriers, "dolphin.yml declares no Adler-32 hash")
sentence = self._faq().split("Adler-32", 1)[1].split("\n\n", 1)[0]
for name in carriers:
self.assertIn(name, sentence, f"FAQ omits the Adler-32 file {name}")
self.assertNotIn(
"IPL.bin", carriers,
"IPL.bin gained an Adler-32 hash; the docs say it has none",
)
for page in ("faq.md", "profiling.md"):
text = (ROOT / "wiki" / page).read_text(encoding="utf-8")
for line in text.splitlines():
if "Adler-32" in line or "adler32" in line:
self.assertNotIn(
"IPL", line,
f"wiki/{page} ties Adler-32 back to IPL: {line.strip()}",
)
def test_existence_platforms_are_not_told_the_verbose_report_is_the_only_check(self):
self.assertIn(
"DISCREPANCY",
self._faq(),
"the FAQ must name the check the platform report performs itself",
)
self.assertIn(
'result["discrepancy"]',
(ROOT / "scripts" / "verify.py").read_text(encoding="utf-8"),
"verify.py no longer raises the discrepancy the FAQ advertises",
)
class CatalogRatioRegressions(unittest.TestCase):
"""The catalog-matched count must read the same on every surface.
The home page, the provenance page, the README and the stats export each
used to count matches on their own. Two of them dropped the systems
scope, so the site published 566 and 553 for the same quantity, one click
apart, and the export paired the wider number with composition.systems as
its denominator.
"""
_SURFACES = ("scripts/generate_site.py", "scripts/generate_readme.py")
def test_no_surface_counts_matches_on_its_own(self):
for relative in self._SURFACES:
tree = ast.parse((ROOT / relative).read_text(encoding="utf-8"))
inline = [
node.lineno
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and getattr(node.func, "id", "") == "sum"
and "provenance" in ast.dump(node)
]
self.assertEqual(
inline,
[],
f"{relative} counts provenance matches inline at {inline}; "
"call common.count_catalog_matched instead",
)
def test_arcade_and_engine_data_stay_out_of_the_ratio(self):
from scripts.common import compute_composition, count_catalog_matched
db = {
"files": {
"a": {"path": "bios/Sony/PlayStation/scph5501.bin", "size": 1,
"provenance": {"redump": {}}},
"b": {"path": "bios/Arcade/Arcade/neogeo.zip", "size": 1,
"provenance": {"no-intro": {}}},
"c": {"path": "bios/ScummVM/soundfonts/Roland.sf2", "size": 1,
"provenance": {"tosec": {}}},
"d": {"path": "bios/Sega/Saturn/sega_101.bin", "size": 1},
}
}
self.assertEqual(count_catalog_matched(db), 1)
self.assertEqual(compute_composition(db)["systems"]["files"], 2)
def test_engine_and_publisher_trees_are_game_data(self):
"""Engine data collected under `Game Engines/` and under a game
publisher counted as console and computer system files: 7 697 of
them made the README announce 13 564 system files."""
from scripts.common import composition_tier
for path in (
"bios/Game Engines/C-Dogs SDL/cdogs/data/guns.json",
"bios/Id Software/Quake III Arena/baseq3/pak1.pk3",
"bios/Epic MegaGames/Jazz Jackrabbit 2/jazz2/Source/share.j2e",
"bios/RPG Maker/easyrpg/rtp/2000/Backdrop/Bridge.png",
):
self.assertEqual(composition_tier(path), "game_data", path)
self.assertEqual(composition_tier("bios/Sony/PlayStation/scph5501.bin"), "systems")
self.assertEqual(composition_tier("bios/Microsoft/MSX/MSX2.ROM"), "systems")
def test_readme_ratio_matches_the_database(self):
from scripts.common import compute_composition, count_catalog_matched, load_database
database = ROOT / "database.json"
if not database.exists():
self.skipTest("database.json not generated")
db = load_database(str(database))
readme = (ROOT / "README.md").read_text(encoding="utf-8")
match = re.search(
r"\*\*([\d,]+) of ([\d,]+) system files\*\* matched to", readme
)
self.assertIsNotNone(match, "README no longer states the catalog ratio")
matched, total = (int(g.replace(",", "")) for g in match.groups())
self.assertEqual(matched, count_catalog_matched(db))
self.assertEqual(total, compute_composition(db)["systems"]["files"])
class PipelineRegressions(unittest.TestCase):
def test_pack_parser_removes_source_metadata(self):
output = "\n".join(
[
"Generating pack for RetroArch [source=full]...",
" 3 files packed (2 baseline + 1 from cores), 2/2 files OK",
]
)
self.assertEqual(pipeline.parse_pack_counts(output), {"RetroArch": (2, 2)})
def test_missing_pack_is_a_consistency_failure(self):
verify = "RetroArch: 2/2 OK"
with contextlib.redirect_stdout(io.StringIO()):
self.assertFalse(pipeline.check_consistency(verify, ""))
def test_display_name_matches_compact_registry_id(self):
verify = "MiSTer FPGA: 65/65 OK [md5]"
pack = "\n".join(
[
"Generating pack for misterfpga [source=full]...",
" pack.zip: 65 files packed (65 baseline + 0 from cores), "
"65/65 files OK [md5]",
]
)
with contextlib.redirect_stdout(io.StringIO()):
self.assertTrue(pipeline.check_consistency(verify, pack))
def test_native_existence_and_strict_pack_exclusions_are_consistent(self):
verify = "RetroArch: 3/3 present [existence]"
pack = "\n".join(
[
"Generating pack for RetroArch [source=full]...",
" pack.zip: 2 files packed (2 baseline + 0 from cores), "
"2/3 files OK, 1 unsafe excluded [existence]",
]
)
with contextlib.redirect_stdout(io.StringIO()):
self.assertTrue(pipeline.check_consistency(verify, pack))
self.assertEqual(pipeline.parse_pack_exclusions(pack), {"RetroArch": 1})
def test_unaccounted_pack_omission_is_a_consistency_failure(self):
verify = "RetroArch: 3/3 present [existence]"
pack = "\n".join(
[
"Generating pack for RetroArch [source=full]...",
" pack.zip: 1 files packed (1 baseline + 0 from cores), "
"1/3 files OK, 1 unsafe excluded, 1 missing [existence]",
]
)
with contextlib.redirect_stdout(io.StringIO()):
self.assertFalse(pipeline.check_consistency(verify, pack))
def test_nothing_parsed_is_not_consistent(self):
"""A changed output format must not pass with no platform compared."""
with contextlib.redirect_stdout(io.StringIO()):
self.assertFalse(pipeline.check_consistency("RetroArch — 2/2 OK", ""))
self.assertFalse(pipeline.check_consistency("", ""))
def test_pack_without_verify_result_is_a_failure(self):
verify = "RetroArch: 2/2 OK"
pack = "\n".join(
[
"Generating pack for RetroArch [source=full]...",
" a.zip: 2 files packed (2 baseline + 0 from cores), 2/2 files OK",
"Generating pack for Batocera [source=full]...",
" b.zip: 2 files packed (2 baseline + 0 from cores), 2/2 files OK",
]
)
with contextlib.redirect_stdout(io.StringIO()):
self.assertFalse(pipeline.check_consistency(verify, pack))
def test_slot_step_reaches_the_summary(self):
source = Path(pipeline.__file__).read_text(encoding="utf-8")
self.assertIn('results["slots"] = ok', source)
def test_every_refresh_failure_reaches_pipeline_exit_status(self):
for failed_label in (
"2/8 refresh data directories",
"2a refresh MAME hashes",
"2a2 refresh FBNeo hashes",
):
with self.subTest(failed_label=failed_label):
def fake_run(_command, label):
return label != failed_label, "RetroArch: 1/1 OK\n"
argv = ["pipeline.py", "--skip-packs", "--skip-docs"]
with (
mock.patch.object(sys, "argv", argv),
mock.patch.object(pipeline, "run", side_effect=fake_run),
contextlib.redirect_stdout(io.StringIO()),
self.assertRaises(SystemExit) as raised,
):
pipeline.main()
self.assertEqual(raised.exception.code, 1)
class ResolverRegressions(unittest.TestCase):
def _database(self, entries: dict[str, Path], suffix: str | None = None) -> dict:
files = {}
by_name: dict[str, list[str]] = {}
by_suffix: dict[str, list[str]] = {}
for name, path in entries.items():
payload = path.read_bytes()
sha1 = hashlib.sha1(payload).hexdigest()
md5 = hashlib.md5(payload).hexdigest()
sha256 = hashlib.sha256(payload).hexdigest()
files[sha1] = {
"path": str(path),
"name": name,
"size": len(payload),
"md5": md5,
"sha256": sha256,
"crc32": "00000000",
}
by_name.setdefault(name, []).append(sha1)
if suffix and name == "firmware.bin":
by_suffix.setdefault(suffix, []).append(sha1)
return {
"files": files,
"indexes": {
"by_name": by_name,
"by_md5": {entry["md5"]: sha1 for sha1, entry in files.items()},
"by_sha256": {
entry["sha256"]: sha1 for sha1, entry in files.items()
},
"by_crc32": {},
"by_path_suffix": by_suffix,
},
}
def test_hash_identity_wins_over_wrong_destination_hint(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
wrong = root / "wrong" / "firmware.bin"
correct = root / "correct" / "firmware.bin"
wrong.parent.mkdir()
correct.parent.mkdir()
wrong.write_bytes(b"wrong")
correct.write_bytes(b"correct")
db = self._database(
{"firmware.bin": wrong, "correct-name.bin": correct},
suffix="Console/USA/firmware.bin",
)
expected = hashlib.sha1(b"correct").hexdigest()
path, status = resolve_local_file(
{"name": "firmware.bin", "sha1": expected},
db,
dest_hint="Console/USA/firmware.bin",
)
self.assertEqual(path, str(correct))
self.assertEqual(status, "sha1_exact")
def test_name_cannot_mask_a_declared_hash_mismatch(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
path = Path(directory) / "firmware.bin"
path.write_bytes(b"wrong")
db = self._database({"firmware.bin": path})
resolved, status = resolve_local_file(
{"name": "firmware.bin", "sha1": "f" * 40}, db
)
self.assertEqual(resolved, str(path))
self.assertEqual(status, "hash_mismatch")
class PackExclusionRegressions(unittest.TestCase):
def test_slug_platform_core_requirement_uses_the_generated_destination(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
platforms = root / "platforms"
platforms.mkdir()
core_payload = root / "extra.bin"
core_payload.write_bytes(b"mapped core payload")
core_sha1 = hashlib.sha1(core_payload.read_bytes()).hexdigest()
core_md5 = hashlib.md5(core_payload.read_bytes()).hexdigest()
database = {
"files": {
core_sha1: {
"name": "extra.bin",
"path": str(core_payload),
"md5": core_md5,
"size": core_payload.stat().st_size,
}
},
"indexes": {
"by_name": {"extra.bin": [core_sha1]},
"by_md5": {core_md5: core_sha1},
"by_sha256": {},
"by_crc32": {},
"by_path_suffix": {},
},
}
config = {
"platform": "Slug platform",
"base_destination": "bios",
"verification_mode": "existence",
"cores": ["core_a"],
"systems": {
"console-a": {
"files": [
{
"name": "base-a.bin",
"destination": "slug-a/base-a.bin",
}
]
},
"console-b": {
"files": [
{
"name": "base-b.bin",
"destination": "slug-b/base-b.bin",
}
]
},
},
}
(platforms / "slug.yml").write_text(
yaml.safe_dump(config), encoding="utf-8"
)
profiles = {
"core_a": {
"emulator": "Core A",
"type": "libretro",
"systems": ["console-a"],
"files": [
{
"name": "extra.bin",
"path": "extra.bin",
"sha1": core_sha1,
}
],
}
}
pack = root / "pack.zip"
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("slug-a/base-a.bin", b"baseline a")
archive.writestr("slug-b/base-b.bin", b"baseline b")
archive.writestr("slug-a/extra.bin", core_payload.read_bytes())
result = verify_pack_against_platform(
str(pack),
"slug",
str(platforms),
db=database,
emu_profiles=profiles,
)
self.assertTrue(result[0], result[3])
self.assertEqual(result[3], [])
self.assertEqual(result[6:8], (1, 1))
def _mismatch_fixture(self, root: Path, mode: str) -> tuple[dict, Path]:
"""A platform declaring a hash the only local payload contradicts."""
platforms = root / "platforms"
platforms.mkdir()
payload = root / "firmware.bin"
payload.write_bytes(b"wrong local variant")
sha1 = hashlib.sha1(payload.read_bytes()).hexdigest()
md5 = hashlib.md5(payload.read_bytes()).hexdigest()
database = {
"files": {
sha1: {
"name": "firmware.bin",
"path": str(payload),
"md5": md5,
"size": payload.stat().st_size,
}
},
"indexes": {
"by_name": {"firmware.bin": [sha1]},
"by_md5": {md5: sha1},
"by_sha256": {},
"by_crc32": {},
"by_path_suffix": {},
},
}
config = {
"platform": f"Platform {mode}",
"verification_mode": mode,
"systems": {
"console": {
"files": [
{
"name": "firmware.bin",
"destination": "firmware.bin",
"sha1": "f" * 40,
}
]
}
},
}
(platforms / "plat.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
return database, platforms
def test_hash_platform_accounts_for_an_unsafe_exclusion(self):
"""A platform that reads the bytes would reject the local payload."""
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
database, platforms = self._mismatch_fixture(root, "md5")
pack = root / "pack.zip"
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("README.txt", "safe subset")
result = verify_pack_against_platform(
str(pack),
"plat",
str(platforms),
db=database,
emu_profiles={},
)
self.assertTrue(result[0], result[3])
self.assertEqual(result[3], [])
self.assertEqual(result[8], 1)
def test_existence_platform_never_withholds_over_a_declared_hash(self):
"""RetroArch and friends only look for the filename.
An upstream hash the local dump contradicts must not remove a file the
frontend would have loaded, so its absence stays a conformance error.
"""
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
database, platforms = self._mismatch_fixture(root, "existence")
pack = root / "pack.zip"
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("README.txt", "no firmware")
result = verify_pack_against_platform(
str(pack),
"plat",
str(platforms),
db=database,
emu_profiles={},
)
self.assertFalse(result[0])
self.assertTrue(any("baseline missing" in e for e in result[3]), result[3])
self.assertEqual(result[8], 0)
def test_unexplained_missing_file_still_fails_conformance(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
platforms = root / "platforms"
platforms.mkdir()
config = {
"platform": "Missing",
"verification_mode": "existence",
"systems": {
"console": {
"files": [
{"name": "absent.bin", "destination": "absent.bin"}
]
}
},
}
(platforms / "missing.yml").write_text(
yaml.safe_dump(config), encoding="utf-8"
)
pack = root / "pack.zip"
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("README.txt", "incomplete")
database = {
"files": {},
"indexes": {
"by_name": {},
"by_md5": {},
"by_sha256": {},
"by_crc32": {},
"by_path_suffix": {},
},
}
result = verify_pack_against_platform(
str(pack),
"missing",
str(platforms),
db=database,
emu_profiles={},
)
self.assertFalse(result[0])
self.assertTrue(any("baseline missing" in error for error in result[3]))
class RegionRegressions(unittest.TestCase):
def test_tagged_and_untagged_same_path_is_preserved(self):
profiles = {
"tagged": {
"type": "libretro",
"files": [
{"name": "bios.bin", "path": "sys/bios.bin", "region": ["japan"]}
],
},
"untagged": {
"type": "libretro",
"files": [{"name": "bios.bin", "path": "sys/bios.bin"}],
},
}
index = region.build_region_index(profiles)
self.assertEqual(region.lookup_regions(index, "sys/bios.bin", "bios.bin"), set())
drops = region.resolve_region_drops(
{"system": [("sys/bios.bin", "bios.bin")]},
index,
["north-america"],
)
self.assertEqual(drops, set())
def test_world_candidate_beats_unmatched_regional_fallback(self):
profiles = {
"core": {
"type": "libretro",
"files": [
{"name": "ntsc.bin", "region": ["world"]},
{"name": "pal.bin", "region": ["europe"]},
],
}
}
index = region.build_region_index(profiles)
groups = {"system": [("ntsc.bin", "ntsc.bin"), ("pal.bin", "pal.bin")]}
self.assertEqual(
region.resolve_region_drops(groups, index, ["north-america"]),
{"pal.bin"},
)
self.assertEqual(region.resolve_region_drops(groups, index, ["europe"]), set())
def test_multi_system_emulator_uses_separate_region_groups(self):
profile = {"systems": ["odyssey2", "videopac"]}
north_america = _emulator_region_group(
"o2em", profile, {"name": "o2rom.bin", "system": "odyssey2"}
)
europe = _emulator_region_group(
"o2em", profile, {"name": "c52.bin", "system": "videopac"}
)
self.assertNotEqual(north_america, europe)
def test_region_audit_accepts_list_valued_md5(self):
sha1 = "a" * 40
md5 = "b" * 32
db = {
"files": {sha1: {}},
"indexes": {"by_md5": {md5: sha1}, "by_name": {}},
}
self.assertEqual(
region_audit.resolve_sha1({"name": "bios.bin", "md5": [md5]}, db),
sha1,
)
class ArchiveSecurityRegressions(unittest.TestCase):
def _zip_path(self, directory: Path, name: str = "archive.zip") -> Path:
return directory / name
def test_member_count_limit_is_enforced(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("one.bin", b"1")
handle.writestr("two.bin", b"2")
with self.assertRaisesRegex(ValueError, "members"):
safe_extract_zip(str(archive), str(root / "out"), max_members=1)
def test_symlink_member_is_rejected(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
link = zipfile.ZipInfo("link")
link.create_system = 3
link.external_attr = (stat.S_IFLNK | 0o777) << 16
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr(link, "target")
with self.assertRaisesRegex(ValueError, "link or special"):
safe_extract_zip(str(archive), str(root / "out"))
def test_windows_separator_is_a_path_not_a_rejection(self):
"""Archives written on Windows store a backslash separator.
download.py feeds third-party archives to this function, so a legal
Windows path must extract into a subdirectory instead of failing.
"""
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("sub\\rom.bin", b"payload")
out = root / "out"
safe_extract_zip(str(archive), str(out))
self.assertEqual((out / "sub" / "rom.bin").read_bytes(), b"payload")
def test_windows_separator_cannot_smuggle_traversal(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
with zipfile.ZipFile(archive, "w") as handle:
handle.writestr("..\\escaped.bin", b"payload")
with self.assertRaisesRegex(ValueError, "traversal"):
safe_extract_zip(str(archive), str(root / "out"))
def test_high_ratio_method_is_bounded_by_size_not_ratio(self):
"""bzip2 legitimately exceeds the DEFLATE ceiling."""
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
with zipfile.ZipFile(archive, "w", zipfile.ZIP_BZIP2) as handle:
handle.writestr("zeros.bin", bytes(4_000_000))
out = root / "out"
safe_extract_zip(str(archive), str(out))
self.assertEqual((out / "zeros.bin").stat().st_size, 4_000_000)
def test_compression_ratio_limit_is_enforced(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
archive = self._zip_path(root)
with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as handle:
handle.writestr("zeros.bin", bytes(16_384))
with self.assertRaisesRegex(ValueError, "compression ratio"):
safe_extract_zip(
str(archive), str(root / "out"), max_compression_ratio=2
)
class NoMd5IsNotNothingChecked(unittest.TestCase):
"""A declared hash that is not an md5 still contradicts.
verify_entry_md5 returned OK the moment the entry declared no md5, so an
entry whose sha256 the local file contradicts read as covered while the
builder was already excluding it. RetroDECK's dsifirmware.bin declares a
sha256 and no md5.
"""
def setUp(self):
import verify
self.verify = verify
def _entry(self):
return {"name": "dsifirmware.bin", "sha256": "b" * 64}
def test_a_contradicted_sha256_is_not_reported_ok(self):
result = self.verify.verify_entry_md5(
self._entry(), "bios/whatever.bin", "hash_mismatch"
)
self.assertNotEqual(result["status"], self.verify.Status.OK)
self.assertIn("contradicted", result.get("reason", ""))
def test_a_clean_resolution_is_still_ok(self):
result = self.verify.verify_entry_md5(
self._entry(), "bios/whatever.bin", "sha256_exact"
)
self.assertEqual(result["status"], self.verify.Status.OK)
class InstallerBoundaryRegressions(unittest.TestCase):
def _manifest(self, dest: str) -> dict:
return {
"manifest_version": 2,
"platform": "retroarch",
"files": [
{
"dest": dest,
"size": 1,
"sha1": "a" * 40,
"sha256": "b" * 64,
"repo_path": "bios/test.bin",
"cores": None,
}
],
"standalone_copies": [],
}
def test_manifest_destination_traversal_is_rejected(self):
with self.assertRaisesRegex(ValueError, "unsafe"):
install._validate_manifest(self._manifest("../escape"), "retroarch")
def test_manifest_repo_source_is_confined_to_bios(self):
manifest = self._manifest("safe.bin")
manifest["files"][0]["repo_path"] = "scripts/pipeline.py"
with self.assertRaisesRegex(ValueError, "outside bios"):
install._validate_manifest(manifest, "retroarch")
def test_omitted_destination_cannot_overlap_a_download(self):
manifest = self._manifest("safe.bin")
manifest["omitted_files"] = [
{
"dest": "safe.bin",
"name": "safe.bin",
"system": "console",
"required": True,
"reason": "hash_mismatch",
"cores": None,
}
]
manifest["total_omitted"] = 1
with self.assertRaisesRegex(ValueError, "conflicting omitted"):
install._validate_manifest(manifest, "retroarch")
def test_omitted_destination_traversal_is_rejected(self):
manifest = self._manifest("safe.bin")
manifest["omitted_files"] = [
{
"dest": "../unsafe.bin",
"name": "unsafe.bin",
"system": "console",
"required": True,
"reason": "hash_mismatch",
"cores": None,
}
]
manifest["total_omitted"] = 1
with self.assertRaisesRegex(ValueError, "unsafe"):
install._validate_manifest(manifest, "retroarch")
def test_target_schema_accepts_the_null_the_generator_emits(self):
"""The schema and generate_target_manifests must agree on null.
A target with no core list is written as null; a schema that rejects
it turns a valid manifest into a CI failure.
"""
from jsonschema import Draft202012Validator
schema = json.loads(
(ROOT / "schemas" / "target-manifest.schema.json").read_text(
encoding="utf-8"
)
)
validator = Draft202012Validator(schema)
document = {"windows": None, "switch": ["a5200"]}
self.assertEqual(list(validator.iter_errors(document)), [])
def test_pack_manifests_are_read_from_inside_the_archive(self):
"""generate_pack writes manifest.json into the ZIP, not beside it.
A filesystem glob over dist/ matches nothing and reports success
without validating a single document.
"""
import validate_schemas
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
dist = Path(directory)
broken = {
"schema_version": 1,
"version": 1,
"generator": "retrobios generate_pack.py",
"generated": "2026-08-10T00:00:00Z",
"files": [{"path": "a.bin", "sha1": "nope", "md5": "nope",
"size": 1, "status": "verified", "name": "a.bin"}],
"summary": {"total_files": 1, "verified": 1, "untracked": 0,
"errors": 0},
"errors": [],
}
with zipfile.ZipFile(dist / "Pack.zip", "w") as archive:
archive.writestr("manifest.json", json.dumps(broken))
errors = validate_schemas._validate_pack_manifests(dist)
self.assertTrue(errors, "an invalid in-archive manifest must be reported")
self.assertTrue(any("sha1" in message for message in errors), errors)
def test_null_core_list_keeps_the_other_targets(self):
"""A target without a core inventory must not void the manifest.
generate_target_manifests emits null for a target that publishes no
core list; rejecting the document would silently disable --target for
every target on that platform.
"""
normalized = install._validate_targets({"windows": None, "switch": ["a5200"]})
self.assertIsNone(normalized["windows"]["cores"])
self.assertEqual(normalized["switch"]["cores"], ["a5200"])
def test_legacy_target_lists_are_normalized(self):
self.assertEqual(
install._validate_targets({"rpi4": ["core-a", "core-b"]}),
{"rpi4": {"cores": ["core-a", "core-b"]}},
)
class TargetManifestRegressions(unittest.TestCase):
def test_yaml_scalar_core_is_rejected_instead_of_leaking_to_json(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
source = root / "source"
output = root / "output"
source.mkdir()
(source / "platform.yml").write_text(
"targets:\n device:\n cores: [81, valid-core]\n",
encoding="utf-8",
)
with self.assertRaisesRegex(ValueError, "non-empty strings"):
generate_target_manifests(str(source), str(output))
class CheckoutCompletenessRegressions(unittest.TestCase):
"""Coverage is resolved against the disk, so the checkout must be whole.
Files over 50 MB and the data directory caches are gitignored. A job that
regenerates the README or the site without restoring them counts those
files as missing and publishes a coverage the collection does not have.
"""
def _steps(self, workflow: str, job: str) -> list[dict]:
data = yaml.safe_load((ROOT / ".github" / "workflows" / workflow).read_text())
return data["jobs"][job]["steps"]
def _index(self, steps: list[dict], needle: str) -> int:
for position, step in enumerate(steps):
if needle in step.get("name", "") or needle in str(step.get("run", "")):
return position
self.fail(f"no step matching {needle!r}")
def test_site_deploy_completes_the_checkout_before_generating(self):
steps = self._steps("deploy-site.yml", "build")
generate = self._index(steps, "Generate site")
self.assertLess(self._index(steps, "restore_large_files.py"), generate)
self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate)
def test_restore_matches_assets_by_content_not_by_name(self):
from scripts.restore_large_files import restore
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
cache = root / "cache"
cache.mkdir()
payload = b"firmware bytes"
(cache / "renamed-asset.bin").write_bytes(payload)
sha1 = hashlib.sha1(payload).hexdigest()
(root / ".gitignore").write_text("bios/Sony/big.pup\n", encoding="utf-8")
(root / "database.json").write_text(
json.dumps({"files": {sha1: {"path": "bios/Sony/big.pup"}}}),
encoding="utf-8",
)
cwd = os.getcwd()
os.chdir(root)
try:
restored, unsatisfied = restore(
str(cache), "database.json", ".gitignore"
)
self.assertEqual((restored, unsatisfied), (1, []))
self.assertEqual((root / "bios/Sony/big.pup").read_bytes(), payload)
# A path already in the checkout is never overwritten.
restored, unsatisfied = restore(
str(cache), "database.json", ".gitignore"
)
self.assertEqual((restored, unsatisfied), (0, []))
finally:
os.chdir(cwd)
def test_restore_leaves_tracked_paths_alone(self):
from scripts.restore_large_files import restore
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
cache = root / "cache"
cache.mkdir()
payload = b"tracked bytes"
(cache / "asset.bin").write_bytes(payload)
sha1 = hashlib.sha1(payload).hexdigest()
(root / ".gitignore").write_text("bios/other.bin\n", encoding="utf-8")
(root / "database.json").write_text(
json.dumps({"files": {sha1: {"path": "bios/tracked.bin"}}}),
encoding="utf-8",
)
cwd = os.getcwd()
os.chdir(root)
try:
restored, unsatisfied = restore(
str(cache), "database.json", ".gitignore"
)
self.assertEqual((restored, unsatisfied), (0, []))
self.assertFalse((root / "bios/tracked.bin").exists())
finally:
os.chdir(cwd)
class EveryManifestEntryIsFetchable(unittest.TestCase):
"""An install manifest may not list a file the installer cannot get.
install.py fetches a file either from its repo_path or from a release
asset. An entry carrying neither is a line in the download list that can
only ever fail. It happened when resolution landed on a file the database
does not index, so the hash was computed but the repo lookup came back
empty and the entry shipped anyway.
"""
def test_committed_manifests_all_have_a_source(self):
manifests = sorted((ROOT / "install").glob("*.json"))
self.assertTrue(manifests, "no install manifests to check")
for path in manifests:
with self.subTest(manifest=path.name):
data = json.loads(path.read_text())
orphans = [
entry["dest"]
for entry in data.get("files", [])
if not entry.get("repo_path") and not entry.get("release_asset")
]
self.assertEqual(
orphans, [], f"{path.name} lists unfetchable files: {orphans[:3]}"
)
def test_an_unresolvable_file_is_recorded_as_omitted(self):
"""The reason must be one install.py knows how to report."""
allowed = {"hash_mismatch", "not_found", "external", "user_provided"}
for path in sorted((ROOT / "install").glob("*.json")):
with self.subTest(manifest=path.name):
data = json.loads(path.read_text())
for entry in data.get("omitted_files", []):
self.assertIn(entry.get("reason"), allowed)
def _manifests(self) -> list[Path]:
paths = sorted((ROOT / "install").glob("*.json"))
paths += sorted((ROOT / "install" / "targets").glob("*.json"))
self.assertTrue(paths, "no install manifests to check")
return paths
def test_repo_paths_name_the_file_the_database_holds(self):
"""A manifest written before a file moved sends the installer to 404.
database.json is regenerated whenever a file is refiled; the manifests
are a separate artefact and were left behind once (the two DS
firmware dumps of the 7 September refile). The installer fetches
repo_path from raw.githubusercontent.com, so a stale one is a
download that can never succeed.
"""
database = ROOT / "database.json"
if not database.is_file():
self.skipTest("database.json not generated")
files = json.loads(database.read_text(encoding="utf-8"))["files"]
for path in self._manifests():
with self.subTest(manifest=path.name):
data = json.loads(path.read_text(encoding="utf-8"))
stale = [
(entry["dest"], entry["repo_path"], files.get(entry["sha1"], {}).get("path"))
for entry in data.get("files", [])
if entry.get("repo_path")
and files.get(entry.get("sha1", ""), {}).get("path") != entry["repo_path"]
]
self.assertEqual(stale, [], f"{path.name} names moved files: {stale[:3]}")
def test_release_entries_are_exactly_the_files_git_does_not_hold(self):
"""The installer has two sources, and the manifest must pick the right one.
A committed file is served by the repository; a gitignored one by
the large-files release. Marking a committed file as a release asset
sends the installer to an asset nobody uploaded (sdlpal/5.avi, 83 MB
in git, was announced as release asset `5.avi`). Marking a gitignored
file as a repo file sends it to a raw URL that has no content.
"""
tracked = set(
subprocess.run(
["git", "ls-files", "-z", "--", "bios"],
capture_output=True, text=True, cwd=ROOT, check=True,
).stdout.split("\0")
)
if not tracked:
self.skipTest("not a git checkout")
for path in self._manifests():
with self.subTest(manifest=path.name):
data = json.loads(path.read_text(encoding="utf-8"))
wrong = []
for entry in data.get("files", []):
repo_path = entry.get("repo_path")
if not repo_path:
continue
committed = repo_path in tracked
if bool(entry.get("release_asset")) == committed:
wrong.append((entry["dest"], repo_path, "committed" if committed else "gitignored"))
self.assertEqual(wrong, [], f"{path.name} picks the wrong source: {wrong[:3]}")
class ReleaseAssetCriterion(unittest.TestCase):
"""What decides whether the installer fetches from the release is git, not size.
Files over 50 MB are supposed to be gitignored and uploaded to the
large-files release, but four committed files break that rule and one of
them reached a manifest as a release asset that does not exist. The
truthful question is whether the repository serves the bytes, and the
ledger of what it does not serve is .gitignore.
"""
def _repo(self, directory: str, ignored: str) -> str:
Path(directory, ".gitignore").write_text(ignored + "\n", encoding="utf-8")
return directory
def test_a_large_committed_file_is_served_by_the_repository(self):
from scripts import generate_pack
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = self._repo(directory, "bios/elsewhere.bin")
big = Path(root, "bios", "big.avi")
big.parent.mkdir()
with big.open("wb") as handle:
handle.truncate(60_000_000)
generate_pack._GITIGNORE_ENTRIES = None
try:
self.assertFalse(generate_pack._is_release_asset(str(big), root))
finally:
generate_pack._GITIGNORE_ENTRIES = None
def test_a_gitignored_file_is_served_by_the_release_whatever_its_size(self):
from scripts import generate_pack
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = self._repo(directory, "bios/small.zip")
small = Path(root, "bios", "small.zip")
small.parent.mkdir()
small.write_bytes(b"PK")
generate_pack._GITIGNORE_ENTRIES = None
try:
self.assertTrue(generate_pack._is_release_asset(str(small), root))
finally:
generate_pack._GITIGNORE_ENTRIES = None
class ReleaseAssetsMatchTheCollection(unittest.TestCase):
"""The bytes the release serves must be the bytes the manifests describe.
install.py checks Content-Length against the manifest size before it
downloads, and the manifest size is that of the local copy. Both FBNeo
sample archives were rebuilt locally after their upload and never
re-uploaded: same members, different container, every install of them
failed. The comparison is offline here on a fixture; the script runs it
against the live release.
"""
def test_missing_and_resized_assets_are_reported_and_nothing_else(self):
from scripts import check_release_assets
expected = {
"bios/Other/fbneo/fbneo/samples/twotiger.zip": 154985212,
"bios/Arcade/MAME/MAME 0.174 Arcade XML.dat": 53677408,
"bios/Other/sdlpal/sdlpal/5.avi": 82833972,
}
assets = {
"twotiger.zip": 154888877,
"MAME.0.174.Arcade.XML.dat": 53677408,
"unrelated.zip": 1,
}
findings = check_release_assets.compare(expected, assets)
self.assertEqual(
findings,
[
("missing", "bios/Other/sdlpal/sdlpal/5.avi", 82833972, None),
("size", "bios/Other/fbneo/fbneo/samples/twotiger.zip", 154985212, 154888877),
],
)
def test_expected_assets_are_the_gitignored_database_entries(self):
from scripts import check_release_assets
db = {
"files": {
"a" * 40: {"path": "bios/Other/x/big.zip", "size": 10},
"b" * 40: {"path": "bios/Other/x/small.bin", "size": 2},
}
}
gitignore = "# large\nbios/Other/x/big.zip\ntmp/\n"
self.assertEqual(
check_release_assets.expected_assets(db, gitignore),
{"bios/Other/x/big.zip": 10},
)
def test_release_notes_are_rendered_from_the_collection(self):
"""The release page is written by hand and drifts; it is rendered instead.
Three assets had no row, three rows carried the SHA1 of an asset
that had since been replaced, and the footer counted 41 files for 44.
Hand-written descriptions survive by name; a new row takes its
description from the profile that declares the file; an asset the
collection does not index is listed apart, with its size only.
"""
from scripts import check_release_assets
db = {"files": {
"a" * 40: {"path": "bios/Nintendo/Wii/nand.bin", "size": 553649152},
"b" * 40: {"path": "bios/Other/fbneo/fbneo/samples/twotiger.zip", "size": 154985212},
"c" * 40: {"path": "bios/Other/x/small.bin", "size": 2},
}}
gitignore = "bios/Nintendo/Wii/nand.bin\nbios/Other/fbneo/fbneo/samples/twotiger.zip\n"
assets = {"nand.bin": 553649152, "twotiger.zip": 154985212,
"sdlpal-data.zip": 140773720, "whoopee.zip": 222665810}
previous = (
"## Arcade\n\n| File | Description | Size | SHA1 |\n|---|---|---|---|\n"
"| [twotiger.zip](u) | Two Tigers samples | 148 MB | `" + "0" * 40 + "` |\n"
"| [whoopee.zip](u) | Toaplan Whoopee samples | 212 MB | `" + "9" * 40 + "` |\n"
)
descriptions = {"nand.bin": "BootMii NAND backup"}
bundles = {"sdlpal-data.zip": "SDLPAL Chinese Paladin game data (.mkf archives)"}
body = check_release_assets.render_notes(
db, gitignore, assets, previous, descriptions, bundles, {"sdlpal-data.zip": "d" * 40}
)
self.assertIn("| [twotiger.zip](https://github.com/Abdess/retrobios/releases/download/large-files/twotiger.zip) | Two Tigers samples | 148 MB | `" + "b" * 40 + "` |", body)
self.assertIn("| [nand.bin](https://github.com/Abdess/retrobios/releases/download/large-files/nand.bin) | BootMii NAND backup | 528 MB | `" + "a" * 40 + "` |", body)
self.assertIn("| [sdlpal-data.zip](https://github.com/Abdess/retrobios/releases/download/large-files/sdlpal-data.zip) | SDLPAL Chinese Paladin game data (.mkf archives) | 134 MB | `" + "d" * 40 + "` |", body)
self.assertIn("## Not indexed", body)
self.assertIn("| [whoopee.zip](https://github.com/Abdess/retrobios/releases/download/large-files/whoopee.zip) | Toaplan Whoopee samples | 212 MB |", body)
self.assertNotIn("9" * 40, body)
self.assertNotIn("small.bin", body)
self.assertIn("4 files, 1.0 GB total", body)
self.assertEqual(body.index("## Console firmware"), min(
body.index(h) for h in ("## Console firmware", "## Arcade", "## Not indexed")
))
def test_stale_notes_are_a_finding(self):
from scripts import check_release_assets
findings = check_release_assets.compare({}, {}, notes_current=False)
self.assertEqual(findings, [("notes", "release description", 0, None)])
def test_pipeline_runs_the_check_online_and_marks_it_skipped_offline(self):
source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
self.assertIn("check_release_assets.py", source)
self.assertIn('results["check_release_assets"] = SKIPPED', source)
class PreservedBytesAreNeverNormalised(unittest.TestCase):
"""git must not rewrite a preserved file's line endings.
SHA1 is the primary key of this collection. Git for Windows sets
core.autocrlf=true by default, so without an attribute saying otherwise a
clone there rewrites every file git guesses is text -- shaders, .ini,
.txt and .dat assets under bios/ -- and each one arrives with a different
hash from the one published here.
"""
def _attr(self, path: str) -> str:
import subprocess
out = subprocess.run(
["git", "check-attr", "text", "--", path],
capture_output=True, text=True, cwd=ROOT,
).stdout
return out.rsplit(":", 1)[-1].strip()
def test_gitattributes_exists(self):
self.assertTrue(
(ROOT / ".gitattributes").is_file(),
"without it git guesses, and guesses wrong on Windows",
)
def test_collection_paths_are_exempt_from_normalisation(self):
for path in (
"bios/Sony/PlayStation/scph5501.bin",
"bios/Other/j2me-loader/color.fsh",
"data/anything.txt",
"data/dolphin-sys/config.json",
):
with self.subTest(path=path):
self.assertEqual(self._attr(path), "unset", f"{path} may be rewritten")
def test_generated_artefacts_stay_lf(self):
for path in (
"database.json",
"scripts/dedup.py",
"README.md",
"provenance/redump.json",
):
with self.subTest(path=path):
self.assertEqual(self._attr(path), "set")
class FreshnessGuardMechanics(unittest.TestCase):
"""write_if_changed is what makes `git diff --exit-code` a real check.
deploy-site.yml regenerates README.md and CONTRIBUTING.md and then fails
if git sees a change. That is only a staleness check because a run which
moves nothing but the clock leaves the file untouched; if the comparison
missed a timestamp form, the guard would fail on every run and stop
meaning anything.
"""
def setUp(self):
from common import write_if_changed
self.write_if_changed = write_if_changed
self._tmp = tempfile.TemporaryDirectory()
self.path = os.path.join(self._tmp.name, "page.md")
def tearDown(self):
self._tmp.cleanup()
def test_a_new_file_is_written(self):
self.assertTrue(self.write_if_changed(self.path, "body\n"))
with open(self.path) as handle:
self.assertEqual(handle.read(), "body\n")
def test_identical_content_is_not_rewritten(self):
self.write_if_changed(self.path, "body\n")
self.assertFalse(self.write_if_changed(self.path, "body\n"))
def test_real_change_is_written(self):
self.write_if_changed(self.path, "body\n")
self.assertTrue(self.write_if_changed(self.path, "other\n"))
def test_every_timestamp_form_is_ignored_on_its_own(self):
forms = [
'{{"generated_at": "{}"}}',
'{{"imported_at": "{}"}}',
"*Auto-generated on {}*",
"*Generated on {}*",
'Generated on {}.
',
]
for form in forms:
with self.subTest(form=form):
first = form.format("2026-01-01T00:00:00Z")
second = form.format("2026-09-09T09:09:09Z")
self.write_if_changed(self.path, first)
self.assertFalse(
self.write_if_changed(self.path, second),
f"a clock-only change rewrote the file for {form!r}",
)
def test_a_change_beside_a_moving_timestamp_is_still_written(self):
self.write_if_changed(self.path, "count: 1\n*Generated on A*\n")
self.assertTrue(
self.write_if_changed(self.path, "count: 2\n*Generated on B*\n")
)
def test_the_written_file_keeps_the_new_timestamp_when_content_changed(self):
self.write_if_changed(self.path, "count: 1\n*Generated on A*\n")
self.write_if_changed(self.path, "count: 2\n*Generated on B*\n")
with open(self.path) as handle:
self.assertIn("Generated on B", handle.read())
class PipelineReportsWhatItDid(unittest.TestCase):
"""A step that did not run must not be summarised as OK.
--offline skips the network steps and --with-export gates three more.
Reporting them as OK claims work nobody did, and a run whose export
never happened read exactly like one where it had.
"""
def test_a_skipped_step_is_not_reported_as_done(self):
import pipeline
self.assertTrue(bool(pipeline.SKIPPED), "a skip must not fail the run")
self.assertEqual(repr(pipeline.SKIPPED), "SKIPPED")
self.assertIsNot(pipeline.SKIPPED, True)
def test_every_skip_branch_uses_the_sentinel(self):
source = (ROOT / "scripts" / "pipeline.py").read_text()
stale = re.findall(r'results\["(\w+)"\] = True', source)
self.assertEqual(stale, [], f"steps still claiming OK when skipped: {stale}")
class DeclaredDependenciesMatchTheDocumentedWorkflows(unittest.TestCase):
"""What the workflows install has to be what the project declares.
mkdocs-material and pymdown-extensions were hard requirements of the site
build in CI and in the release guide while pyproject declared neither, so
a contributor following either had to read the workflow to find out.
"""
@staticmethod
def _pyproject() -> dict:
import tomllib
with (ROOT / "pyproject.toml").open("rb") as handle:
return tomllib.load(handle)
def test_every_package_ci_installs_is_declared(self):
extras = self._pyproject()["project"]["optional-dependencies"]
declared = {
name.split(">")[0].split("=")[0].split("<")[0].strip('"')
for group in extras.values()
for name in group
}
declared |= set(self._pyproject()["project"]["dependencies"])
installed: set[str] = set()
for workflow in ("validate.yml", "deploy-site.yml"):
body = (ROOT / ".github" / "workflows" / workflow).read_text(
encoding="utf-8"
)
for line in body.splitlines():
if "pip install" not in line:
continue
for token in line.split("pip install", 1)[1].split():
name = token.strip('"').split(">")[0].split("=")[0]
name = name.split("<")[0].strip()
if name:
installed.add(name)
self.assertEqual(
installed - declared,
set(),
"CI installs packages pyproject does not declare",
)
class ACheckThatCannotAnswerDoesNotPass(unittest.TestCase):
"""Exiting zero says the question was answered and the answer was yes.
Four scripts said that without answering: a refresh that reached no
remote, a freshness check whose upstream was unreachable, a restore whose
cache could not supply a declared path, and a pack verification asked
about one platform whose pack was not there.
"""
def test_naming_a_platform_with_no_pack_is_not_a_pass(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
named = subprocess.run(
[sys.executable, "scripts/generate_pack.py", "--platform",
"retroarch", "--verify-packs", "--output-dir", directory],
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
)
self.assertNotEqual(
named.returncode, 0,
"a named platform with no pack reported success:\n"
+ named.stdout + named.stderr,
)
# --all is a sweep: a platform nobody built is out of scope.
swept = subprocess.run(
[sys.executable, "scripts/generate_pack.py", "--all",
"--verify-packs", "--output-dir", directory],
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
)
self.assertEqual(swept.returncode, 0, swept.stdout + swept.stderr)
def test_an_unsatisfiable_declared_path_is_reported(self):
from scripts.restore_large_files import restore
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
cache = root / "cache"
cache.mkdir()
(root / ".gitignore").write_text("bios/absent.bin\n", encoding="utf-8")
(root / "database.json").write_text(
json.dumps({"files": {"a" * 40: {"path": "bios/absent.bin"}}}),
encoding="utf-8",
)
cwd = os.getcwd()
os.chdir(root)
try:
restored, unsatisfied = restore(
str(cache), "database.json", ".gitignore"
)
finally:
os.chdir(cwd)
self.assertEqual(restored, 0)
self.assertEqual(unsatisfied, ["bios/absent.bin"])
def test_an_unreachable_buildbot_is_not_a_fresh_verdict(self):
source = (ROOT / "scripts" / "check_buildbot_system.py").read_text(
encoding="utf-8"
)
self.assertIn(
'if report.get("error"):',
source.split("def main(")[-1],
"main() ignores the error the report carries",
)
def test_a_missing_profile_directory_says_so(self):
import io as _io
from scripts import common as _common
stderr = _io.StringIO()
with contextlib.redirect_stderr(stderr):
profiles = _common.load_emulator_profiles(
str(ROOT / "no-such-emulator-dir")
)
self.assertEqual(profiles, {})
self.assertIn("no emulator profile directory", stderr.getvalue())
class TestEntryPointsRunEveryClass(unittest.TestCase):
"""`unittest.main()` has to sit after the last test class.
In this file it sat in the middle, so running it directly discovered only
the classes defined above it: eight cases never ran that way, while
`python -m unittest discover` ran all of them. The two roads have to agree.
"""
def test_no_module_calls_main_before_its_last_class(self):
for path in sorted((ROOT / "tests").glob("test_*.py")):
lines = path.read_text(encoding="utf-8").splitlines()
# Column zero only: the same text appears inside this very test as
# a string literal, and a substring search matched that instead.
entry = next(
(n for n, line in enumerate(lines)
if line.startswith("if __name__ ==")),
None,
)
if entry is None:
continue
last_class = max(
(n for n, line in enumerate(lines) if line.startswith("class ")),
default=-1,
)
self.assertGreater(
entry,
last_class,
f"{path.name} calls unittest.main() before its last class, so "
"running the file directly skips what follows",
)
class ScriptsImportThreeWays(unittest.TestCase):
"""A script is run directly, run as a module, and imported as a package.
The three do not agree on what is on the path: only the first form adds
the scripts directory. Adding the package marker without the bootstrap
made `import scripts.common` fail on the first sibling import it reached.
"""
def _run(self, *args: str):
import subprocess
return subprocess.run(
[sys.executable, *args],
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
)
def test_imported_as_a_package(self):
result = self._run(
"-c", "import scripts.common, scripts.verify, scripts.generate_pack"
)
self.assertEqual(result.returncode, 0, result.stderr[-400:])
def test_run_as_a_module(self):
result = self._run("-m", "scripts.scraper.libretro_scraper", "--help")
self.assertEqual(result.returncode, 0, result.stderr[-400:])
def test_run_as_a_script(self):
result = self._run("scripts/list_platforms.py")
self.assertEqual(result.returncode, 0, result.stderr[-400:])
class ContributorsSurviveAFailedRequest(unittest.TestCase):
"""The one part of the README that comes from the network.
A refused or rate-limited request left the list empty, which deleted the
section from a published README. It happened during a pipeline run and
the result was committed, after which the freshness check regenerated the
section and failed on the difference. Losing the list is a worse answer
than publishing a stale one.
"""
def _block(self, text: str):
import tempfile
sys.path.insert(0, str(ROOT / "scripts"))
from generate_readme import _existing_contributor_block
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as handle:
handle.write(text)
name = handle.name
try:
return _existing_contributor_block(name)
finally:
os.unlink(name)
def test_a_published_list_is_read_back(self):
text = (
"# Title\n\n## Contributors\n\n"
'
\n'
'
\n\n'
"## Community tools\n\nsomething else\n"
)
block = self._block(text)
self.assertEqual(block[0], "## Contributors")
self.assertIn('alt="a"', "\n".join(block))
self.assertIn('alt="b"', "\n".join(block))
self.assertNotIn("Community tools", "\n".join(block))
self.assertNotIn("something else", "\n".join(block))
def test_a_readme_without_the_section_yields_nothing(self):
self.assertEqual(self._block("# Title\n\n## Other\n\ntext\n"), [])
def test_an_unreadable_file_yields_nothing(self):
sys.path.insert(0, str(ROOT / "scripts"))
from generate_readme import _existing_contributor_block
self.assertEqual(_existing_contributor_block("/nonexistent/README.md"), [])
def test_the_committed_readme_still_carries_its_contributors(self):
"""Regenerating offline must never be the reason this disappears."""
text = (ROOT / "README.md").read_text()
if "## Contributors" not in text:
self.skipTest("README carries no contributors section")
self.assertGreater(
text.count(' list[str]:
seen: dict[str, str] = {}
again = []
for node in ast.parse(path.read_text()).body:
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
continue
target = node.targets[0]
if not isinstance(target, ast.Name):
continue
value = ast.unparse(node.value)
if seen.get(target.id) == value:
again.append(target.id)
seen[target.id] = value
return again
def test_no_module_assigns_the_same_constant_twice(self):
offenders = {}
for path in sorted(ROOT.glob("scripts/**/*.py")) + [ROOT / "install.py"]:
again = self._redeclared(path)
if again:
offenders[path.name] = again
self.assertEqual(offenders, {})
if __name__ == "__main__":
unittest.main()