mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 21:43:23 -05:00
1886 lines
77 KiB
Python
1886 lines
77 KiB
Python
"""Regression tests for the holistic reliability and security audit."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import zipfile
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import yaml
|
|
|
|
try:
|
|
import jsonschema # noqa: F401
|
|
except ImportError:
|
|
HAS_JSONSCHEMA = False
|
|
else:
|
|
HAS_JSONSCHEMA = True
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
TMP_ROOT = ROOT / "tmp" / "tests"
|
|
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
sys.path.insert(0, str(ROOT))
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
import install
|
|
from scripts import pipeline, region, region_audit
|
|
from scripts.common import resolve_local_file, safe_extract_zip
|
|
from scripts.generate_pack import (
|
|
_emulator_region_group,
|
|
generate_target_manifests,
|
|
verify_pack_against_platform,
|
|
)
|
|
|
|
|
|
class ReadmeRegressions(unittest.TestCase):
|
|
"""The README advertises commands; these must stay executable.
|
|
|
|
Wording is the maintainer's, so nothing here asserts prose. What is
|
|
asserted is that every flag and URL the README hands a reader is one the
|
|
shipped scripts actually accept.
|
|
"""
|
|
|
|
def _quick_install(self) -> str:
|
|
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
|
return readme.split("## Quick Install", 1)[1].split(
|
|
"## Download BIOS packs", 1
|
|
)[0]
|
|
|
|
def test_advertised_bootstrap_urls_are_the_shipped_ones(self):
|
|
quick_install = self._quick_install()
|
|
for script in ("install.sh", "install.ps1"):
|
|
url = f"https://raw.githubusercontent.com/Abdess/retrobios/main/{script}"
|
|
self.assertIn(url, quick_install, f"{script} bootstrap URL missing")
|
|
self.assertIn(
|
|
"https://raw.githubusercontent.com/Abdess/retrobios/main/install.py",
|
|
(ROOT / "install.sh").read_text(encoding="utf-8"),
|
|
"install.sh must fetch install.py from the ref the README advertises",
|
|
)
|
|
|
|
def test_advertised_installer_flags_exist(self):
|
|
quick_install = self._quick_install()
|
|
advertised = set(re.findall(r"(?<![\w-])--[a-z][a-z-]+", quick_install))
|
|
parser_source = (ROOT / "install.py").read_text(encoding="utf-8")
|
|
supported = set(re.findall(r'add_argument\(\s*"(--[a-z][a-z-]+)"', parser_source))
|
|
unknown = advertised - supported
|
|
self.assertEqual(unknown, set(), f"README advertises unknown flags: {unknown}")
|
|
|
|
|
|
class ReleaseSigningRegressions(unittest.TestCase):
|
|
"""A checksum list published beside its own artifacts proves nothing.
|
|
|
|
SHA256SUMS.txt answers corruption; whoever can rewrite a release rewrites
|
|
the list with it. The signature is what a third party checks, and it is
|
|
only checkable while allowed_signers, the signing step and the upload
|
|
list stay in agreement.
|
|
"""
|
|
|
|
PRINCIPAL = "releases@retrobios"
|
|
|
|
def test_allowed_signers_names_one_usable_key(self):
|
|
path = ROOT / "allowed_signers"
|
|
self.assertTrue(path.exists(), "allowed_signers is the published trust root")
|
|
lines = [
|
|
line for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line.strip() and not line.startswith("#")
|
|
]
|
|
self.assertTrue(lines, "allowed_signers carries no key")
|
|
for line in lines:
|
|
principal, keytype, blob = line.split()[:3]
|
|
self.assertEqual(principal, self.PRINCIPAL)
|
|
self.assertTrue(keytype.startswith(("ssh-", "sk-", "ecdsa-")), keytype)
|
|
self.assertTrue(len(blob) > 40, "key material looks truncated")
|
|
|
|
def test_the_release_signs_the_list_and_ships_the_signature(self):
|
|
process = (ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8")
|
|
self.assertIn(
|
|
"ssh-keygen -Y sign", process, "the release no longer signs the list"
|
|
)
|
|
upload = next(
|
|
line for line in process.splitlines()
|
|
if line.startswith("for f in dist/SHA256SUMS.txt")
|
|
)
|
|
self.assertIn(
|
|
"dist/SHA256SUMS.txt.sig", upload,
|
|
"the signature is produced but never uploaded",
|
|
)
|
|
|
|
def test_the_documented_verification_matches_the_published_principal(self):
|
|
process = (ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8")
|
|
self.assertIn("ssh-keygen -Y verify", process)
|
|
self.assertIn(f"-I {self.PRINCIPAL}", process)
|
|
self.assertIn("allowed_signers", process)
|
|
|
|
|
|
class WorkflowRegressions(unittest.TestCase):
|
|
"""The test suite must sit on every road into main.
|
|
|
|
run-tests lived in a pull_request-only workflow while the work landed by
|
|
direct push, so 1,318 cases guarded a road almost nothing took. What is
|
|
asserted here is reachability, not the workflow's shape.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _workflow(name: str) -> dict:
|
|
with (ROOT / ".github" / "workflows" / name).open(encoding="utf-8") as f:
|
|
document = yaml.safe_load(f)
|
|
# PyYAML resolves the bare `on:` key to the boolean True.
|
|
document["on"] = document.pop(True, document.get("on"))
|
|
return document
|
|
|
|
def test_no_validation_step_discards_its_exit_code(self):
|
|
"""A check whose result is thrown away is not a check.
|
|
|
|
The BIOS validation step ended in `|| true`, so validate_pr.py could
|
|
exit 1 on a file that failed its hash and the job stayed green. The
|
|
report still has to reach the pull request, so the code is recorded
|
|
and acted on afterwards rather than swallowed. A best-effort side
|
|
action such as adding a label is not a check and keeps its `|| true`.
|
|
"""
|
|
checks = ("python scripts/", "unittest", "mkdocs build")
|
|
for name in ("validate.yml", "deploy-site.yml"):
|
|
workflow = self._workflow(name)
|
|
for job_name, job in workflow["jobs"].items():
|
|
for step in job.get("steps", []):
|
|
# A shell continuation puts the command and its `|| true`
|
|
# on different lines, so they are rejoined before scanning.
|
|
body = str(step.get("run", "")).replace("\\\n", " ")
|
|
for line in body.splitlines():
|
|
if not any(marker in line for marker in checks):
|
|
continue
|
|
self.assertNotIn(
|
|
"|| true",
|
|
line,
|
|
f"{name}:{job_name}:{step.get('name', '?')} runs a "
|
|
f"check and discards its exit code: {line.strip()}",
|
|
)
|
|
|
|
workflow = self._workflow("validate.yml")
|
|
steps = workflow["jobs"]["validate-bios"]["steps"]
|
|
gate = [s for s in steps if "rc != " in str(s.get("if", ""))]
|
|
self.assertTrue(
|
|
gate,
|
|
"nothing in validate-bios acts on the validation exit code",
|
|
)
|
|
|
|
def test_the_suite_runs_on_a_direct_push_to_main(self):
|
|
workflow = self._workflow("validate.yml")
|
|
triggers = workflow["on"]
|
|
self.assertIn("push", triggers, "validate.yml no longer runs on push")
|
|
self.assertIn("main", triggers["push"]["branches"])
|
|
job = next(
|
|
(
|
|
name
|
|
for name, body in workflow["jobs"].items()
|
|
if any(
|
|
"unittest discover" in str(step.get("run", ""))
|
|
for step in body.get("steps", [])
|
|
)
|
|
),
|
|
None,
|
|
)
|
|
self.assertIsNotNone(job, "no job runs the test suite")
|
|
self.assertIsNone(
|
|
workflow["jobs"][job].get("if"),
|
|
f"{job} carries a condition; the suite must run on push and on PR",
|
|
)
|
|
|
|
def test_both_events_watch_the_same_paths(self):
|
|
triggers = self._workflow("validate.yml")["on"]
|
|
self.assertEqual(
|
|
triggers["pull_request"]["paths"],
|
|
triggers["push"]["paths"],
|
|
"the two path lists have drifted; a push would skip what a PR checks",
|
|
)
|
|
|
|
def test_pull_request_only_jobs_are_guarded(self):
|
|
workflow = self._workflow("validate.yml")
|
|
for name, body in workflow["jobs"].items():
|
|
uses_pr_context = "github.event.pull_request" in yaml.dump(body)
|
|
if uses_pr_context:
|
|
self.assertEqual(
|
|
body.get("if"),
|
|
"github.event_name == 'pull_request'",
|
|
f"{name} reads pull request context and would run on a push",
|
|
)
|
|
|
|
def test_no_workflow_relies_on_a_yaml_anchor(self):
|
|
"""The workflow parser reads no anchor; PyYAML would hide the break."""
|
|
for path in sorted((ROOT / ".github" / "workflows").glob("*.yml")):
|
|
for number, line in enumerate(
|
|
path.read_text(encoding="utf-8").splitlines(), 1
|
|
):
|
|
self.assertIsNone(
|
|
re.search(r"(?:^|\s)[&*][A-Za-z_]", line),
|
|
f"{path.name}:{number} uses a YAML anchor: {line.strip()}",
|
|
)
|
|
|
|
|
|
class UnreachableCitationRegressions(unittest.TestCase):
|
|
"""A citation must name a place a declared repository can hold.
|
|
|
|
kenji-nx carried tmp/es-de/ANDROID.md:470-474, a path from the machine
|
|
of whoever profiled it. No revision of any declared repository holds it,
|
|
so profile_sync could only report it missing, every pass, forever. The
|
|
check is offline and runs on every push.
|
|
"""
|
|
|
|
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
|
def _hits(self, text: str) -> bool:
|
|
import validate_schemas
|
|
|
|
return bool(validate_schemas._UNREACHABLE_REF.search(" " + text))
|
|
|
|
def test_a_scratch_directory_is_rejected(self):
|
|
self.assertTrue(self._hits("tmp/es-de/ANDROID.md:470-474"))
|
|
|
|
def test_an_absolute_path_is_rejected(self):
|
|
self.assertTrue(self._hits("/home/someone/src/a.c:12"))
|
|
self.assertTrue(self._hits(r"C:\work\src\a.c:12"))
|
|
|
|
def test_a_path_climbing_out_of_the_tree_is_rejected(self):
|
|
self.assertTrue(self._hits("../outside/src/a.c:12"))
|
|
|
|
def test_ordinary_citations_pass(self):
|
|
for good in (
|
|
"src/core/main.cpp:210",
|
|
"libretro.c:5293-5337",
|
|
"PCE.emu/src/main/Main.cc:80-91",
|
|
"es-de ANDROID.md:470-474",
|
|
"see tmp files for details",
|
|
):
|
|
with self.subTest(citation=good):
|
|
self.assertFalse(self._hits(good))
|
|
|
|
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
|
def test_the_corpus_carries_none(self):
|
|
import validate_schemas
|
|
|
|
errors = []
|
|
for path in sorted((ROOT / "emulators").glob("*.yml")):
|
|
with path.open(encoding="utf-8") as handle:
|
|
errors.extend(
|
|
validate_schemas._unreachable_citations(path, yaml.safe_load(handle))
|
|
)
|
|
self.assertEqual(errors, [])
|
|
|
|
|
|
class FaqRegressions(unittest.TestCase):
|
|
"""The FAQ states facts the code owns; these tie it back to the source.
|
|
|
|
Three of its claims had drifted: a pinned MAME version, Adler-32
|
|
attributed to Dolphin's IPL instead of its DSP ROMs, and a verbose
|
|
per-emulator report described as the only way to catch a bad file on an
|
|
existence platform. Readers act on all three.
|
|
"""
|
|
|
|
_MAME_GENERATIONS = {
|
|
"MAME 2000": "mame2000",
|
|
"MAME 2003": "mame2003",
|
|
"MAME 2009": "mame2009",
|
|
"MAME 2010": "mame2010",
|
|
"MAME 2015": "mame2015",
|
|
"MAME 2016": "mame2016",
|
|
"current MAME": "mame",
|
|
}
|
|
|
|
@staticmethod
|
|
def _faq() -> str:
|
|
return (ROOT / "wiki" / "faq.md").read_text(encoding="utf-8")
|
|
|
|
@staticmethod
|
|
def _profile(key: str) -> dict:
|
|
with (ROOT / "emulators" / f"{key}.yml").open(encoding="utf-8") as handle:
|
|
return yaml.safe_load(handle)
|
|
|
|
def test_mame_versions_match_their_profiles(self):
|
|
# The list is wrapped, so a label and its version can straddle a line.
|
|
faq = " ".join(self._faq().split())
|
|
for label, key in self._MAME_GENERATIONS.items():
|
|
match = re.search(rf"{re.escape(label)} \(([^)]+)\)", faq)
|
|
self.assertIsNotNone(match, f"FAQ no longer states a version for {label}")
|
|
self.assertEqual(
|
|
match.group(1),
|
|
self._profile(key)["core_version"],
|
|
f"FAQ version for {label} has drifted from emulators/{key}.yml",
|
|
)
|
|
|
|
def test_adler32_is_attributed_to_the_files_that_carry_it(self):
|
|
dolphin = self._profile("dolphin")
|
|
carriers = {
|
|
f["name"] for f in dolphin["files"] if f.get("known_hash_adler32")
|
|
}
|
|
self.assertTrue(carriers, "dolphin.yml declares no Adler-32 hash")
|
|
sentence = self._faq().split("Adler-32", 1)[1].split("\n\n", 1)[0]
|
|
for name in carriers:
|
|
self.assertIn(name, sentence, f"FAQ omits the Adler-32 file {name}")
|
|
self.assertNotIn(
|
|
"IPL.bin", carriers,
|
|
"IPL.bin gained an Adler-32 hash; the docs say it has none",
|
|
)
|
|
for page in ("faq.md", "profiling.md"):
|
|
text = (ROOT / "wiki" / page).read_text(encoding="utf-8")
|
|
for line in text.splitlines():
|
|
if "Adler-32" in line or "adler32" in line:
|
|
self.assertNotIn(
|
|
"IPL", line,
|
|
f"wiki/{page} ties Adler-32 back to IPL: {line.strip()}",
|
|
)
|
|
|
|
def test_existence_platforms_are_not_told_the_verbose_report_is_the_only_check(self):
|
|
self.assertIn(
|
|
"DISCREPANCY",
|
|
self._faq(),
|
|
"the FAQ must name the check the platform report performs itself",
|
|
)
|
|
self.assertIn(
|
|
'result["discrepancy"]',
|
|
(ROOT / "scripts" / "verify.py").read_text(encoding="utf-8"),
|
|
"verify.py no longer raises the discrepancy the FAQ advertises",
|
|
)
|
|
|
|
|
|
class CatalogRatioRegressions(unittest.TestCase):
|
|
"""The catalog-matched count must read the same on every surface.
|
|
|
|
The home page, the provenance page, the README and the stats export each
|
|
used to count matches on their own. Two of them dropped the systems
|
|
scope, so the site published 566 and 553 for the same quantity, one click
|
|
apart, and the export paired the wider number with composition.systems as
|
|
its denominator.
|
|
"""
|
|
|
|
_SURFACES = ("scripts/generate_site.py", "scripts/generate_readme.py")
|
|
|
|
def test_no_surface_counts_matches_on_its_own(self):
|
|
for relative in self._SURFACES:
|
|
tree = ast.parse((ROOT / relative).read_text(encoding="utf-8"))
|
|
inline = [
|
|
node.lineno
|
|
for node in ast.walk(tree)
|
|
if isinstance(node, ast.Call)
|
|
and getattr(node.func, "id", "") == "sum"
|
|
and "provenance" in ast.dump(node)
|
|
]
|
|
self.assertEqual(
|
|
inline,
|
|
[],
|
|
f"{relative} counts provenance matches inline at {inline}; "
|
|
"call common.count_catalog_matched instead",
|
|
)
|
|
|
|
def test_arcade_and_engine_data_stay_out_of_the_ratio(self):
|
|
from scripts.common import compute_composition, count_catalog_matched
|
|
|
|
db = {
|
|
"files": {
|
|
"a": {"path": "bios/Sony/PlayStation/scph5501.bin", "size": 1,
|
|
"provenance": {"redump": {}}},
|
|
"b": {"path": "bios/Arcade/Arcade/neogeo.zip", "size": 1,
|
|
"provenance": {"no-intro": {}}},
|
|
"c": {"path": "bios/ScummVM/soundfonts/Roland.sf2", "size": 1,
|
|
"provenance": {"tosec": {}}},
|
|
"d": {"path": "bios/Sega/Saturn/sega_101.bin", "size": 1},
|
|
}
|
|
}
|
|
self.assertEqual(count_catalog_matched(db), 1)
|
|
self.assertEqual(compute_composition(db)["systems"]["files"], 2)
|
|
|
|
def test_engine_and_publisher_trees_are_game_data(self):
|
|
"""Engine data collected under `Game Engines/` and under a game
|
|
publisher counted as console and computer system files: 7 697 of
|
|
them made the README announce 13 564 system files."""
|
|
from scripts.common import composition_tier
|
|
|
|
for path in (
|
|
"bios/Game Engines/C-Dogs SDL/cdogs/data/guns.json",
|
|
"bios/Id Software/Quake III Arena/baseq3/pak1.pk3",
|
|
"bios/Epic MegaGames/Jazz Jackrabbit 2/jazz2/Source/share.j2e",
|
|
"bios/RPG Maker/easyrpg/rtp/2000/Backdrop/Bridge.png",
|
|
):
|
|
self.assertEqual(composition_tier(path), "game_data", path)
|
|
self.assertEqual(composition_tier("bios/Sony/PlayStation/scph5501.bin"), "systems")
|
|
self.assertEqual(composition_tier("bios/Microsoft/MSX/MSX2.ROM"), "systems")
|
|
|
|
def test_readme_ratio_matches_the_database(self):
|
|
from scripts.common import compute_composition, count_catalog_matched, load_database
|
|
|
|
database = ROOT / "database.json"
|
|
if not database.exists():
|
|
self.skipTest("database.json not generated")
|
|
db = load_database(str(database))
|
|
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
|
match = re.search(
|
|
r"\*\*([\d,]+) of ([\d,]+) system files\*\* matched to", readme
|
|
)
|
|
self.assertIsNotNone(match, "README no longer states the catalog ratio")
|
|
matched, total = (int(g.replace(",", "")) for g in match.groups())
|
|
self.assertEqual(matched, count_catalog_matched(db))
|
|
self.assertEqual(total, compute_composition(db)["systems"]["files"])
|
|
|
|
|
|
class PipelineRegressions(unittest.TestCase):
|
|
def test_pack_parser_removes_source_metadata(self):
|
|
output = "\n".join(
|
|
[
|
|
"Generating pack for RetroArch [source=full]...",
|
|
" 3 files packed (2 baseline + 1 from cores), 2/2 files OK",
|
|
]
|
|
)
|
|
self.assertEqual(pipeline.parse_pack_counts(output), {"RetroArch": (2, 2)})
|
|
|
|
def test_missing_pack_is_a_consistency_failure(self):
|
|
verify = "RetroArch: 2/2 OK"
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertFalse(pipeline.check_consistency(verify, ""))
|
|
|
|
def test_display_name_matches_compact_registry_id(self):
|
|
verify = "MiSTer FPGA: 65/65 OK [md5]"
|
|
pack = "\n".join(
|
|
[
|
|
"Generating pack for misterfpga [source=full]...",
|
|
" pack.zip: 65 files packed (65 baseline + 0 from cores), "
|
|
"65/65 files OK [md5]",
|
|
]
|
|
)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertTrue(pipeline.check_consistency(verify, pack))
|
|
|
|
def test_native_existence_and_strict_pack_exclusions_are_consistent(self):
|
|
verify = "RetroArch: 3/3 present [existence]"
|
|
pack = "\n".join(
|
|
[
|
|
"Generating pack for RetroArch [source=full]...",
|
|
" pack.zip: 2 files packed (2 baseline + 0 from cores), "
|
|
"2/3 files OK, 1 unsafe excluded [existence]",
|
|
]
|
|
)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertTrue(pipeline.check_consistency(verify, pack))
|
|
self.assertEqual(pipeline.parse_pack_exclusions(pack), {"RetroArch": 1})
|
|
|
|
def test_unaccounted_pack_omission_is_a_consistency_failure(self):
|
|
verify = "RetroArch: 3/3 present [existence]"
|
|
pack = "\n".join(
|
|
[
|
|
"Generating pack for RetroArch [source=full]...",
|
|
" pack.zip: 1 files packed (1 baseline + 0 from cores), "
|
|
"1/3 files OK, 1 unsafe excluded, 1 missing [existence]",
|
|
]
|
|
)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertFalse(pipeline.check_consistency(verify, pack))
|
|
|
|
def test_nothing_parsed_is_not_consistent(self):
|
|
"""A changed output format must not pass with no platform compared."""
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertFalse(pipeline.check_consistency("RetroArch — 2/2 OK", ""))
|
|
self.assertFalse(pipeline.check_consistency("", ""))
|
|
|
|
def test_pack_without_verify_result_is_a_failure(self):
|
|
verify = "RetroArch: 2/2 OK"
|
|
pack = "\n".join(
|
|
[
|
|
"Generating pack for RetroArch [source=full]...",
|
|
" a.zip: 2 files packed (2 baseline + 0 from cores), 2/2 files OK",
|
|
"Generating pack for Batocera [source=full]...",
|
|
" b.zip: 2 files packed (2 baseline + 0 from cores), 2/2 files OK",
|
|
]
|
|
)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertFalse(pipeline.check_consistency(verify, pack))
|
|
|
|
def test_only_the_full_variant_feeds_the_comparison(self):
|
|
"""--all-variants printed six packs under one label; the last won."""
|
|
output = "\n".join([
|
|
"Generating pack for RetroArch [source=full]...",
|
|
" a.zip: 5 files packed (5 baseline + 0 from cores), 535/535 files OK",
|
|
"Generating pack for RetroArch [source=platform, required]...",
|
|
" b.zip: 5 files packed (5 baseline + 0 from cores), 530/530 files OK",
|
|
])
|
|
self.assertEqual(pipeline.parse_pack_counts(output), {"RetroArch": (535, 535)})
|
|
|
|
def test_slot_step_reaches_the_summary(self):
|
|
source = Path(pipeline.__file__).read_text(encoding="utf-8")
|
|
self.assertIn('results["slots"] = ok', source)
|
|
|
|
def test_every_refresh_failure_reaches_pipeline_exit_status(self):
|
|
for failed_label in (
|
|
"2/8 refresh data directories",
|
|
"2a refresh MAME hashes",
|
|
"2a2 refresh FBNeo hashes",
|
|
):
|
|
with self.subTest(failed_label=failed_label):
|
|
def fake_run(_command, label):
|
|
return label != failed_label, "RetroArch: 1/1 OK\n"
|
|
|
|
argv = ["pipeline.py", "--skip-packs", "--skip-docs"]
|
|
with (
|
|
mock.patch.object(sys, "argv", argv),
|
|
mock.patch.object(pipeline, "run", side_effect=fake_run),
|
|
contextlib.redirect_stdout(io.StringIO()),
|
|
self.assertRaises(SystemExit) as raised,
|
|
):
|
|
pipeline.main()
|
|
self.assertEqual(raised.exception.code, 1)
|
|
|
|
|
|
def test_a_targeted_export_reads_a_full_model(self):
|
|
"""--target wrote truth/<target>/ only; the export read truth/."""
|
|
commands: list[list[str]] = []
|
|
|
|
def fake_run(command, _label):
|
|
commands.append([str(part) for part in command])
|
|
return True, "RetroArch: 1/1 OK\n"
|
|
|
|
argv = ["pipeline.py", "--offline", "--skip-packs", "--skip-docs",
|
|
"--with-export", "--target", "switch"]
|
|
with (
|
|
mock.patch.object(sys, "argv", argv),
|
|
mock.patch.object(pipeline, "run", side_effect=fake_run),
|
|
contextlib.redirect_stdout(io.StringIO()),
|
|
):
|
|
with contextlib.suppress(SystemExit):
|
|
pipeline.main()
|
|
truth_runs = [c for c in commands if "scripts/generate_truth.py" in c]
|
|
export = next(c for c in commands if "scripts/export_native.py" in c)
|
|
read_dir = export[export.index("--truth-dir") + 1]
|
|
full_models = [
|
|
c for c in truth_runs
|
|
if "--target" not in c and c[c.index("--output-dir") + 1] == read_dir
|
|
]
|
|
self.assertTrue(full_models, truth_runs)
|
|
|
|
|
|
class ResolverRegressions(unittest.TestCase):
|
|
def _database(self, entries: dict[str, Path], suffix: str | None = None) -> dict:
|
|
files = {}
|
|
by_name: dict[str, list[str]] = {}
|
|
by_suffix: dict[str, list[str]] = {}
|
|
for name, path in entries.items():
|
|
payload = path.read_bytes()
|
|
sha1 = hashlib.sha1(payload).hexdigest()
|
|
md5 = hashlib.md5(payload).hexdigest()
|
|
sha256 = hashlib.sha256(payload).hexdigest()
|
|
files[sha1] = {
|
|
"path": str(path),
|
|
"name": name,
|
|
"size": len(payload),
|
|
"md5": md5,
|
|
"sha256": sha256,
|
|
"crc32": "00000000",
|
|
}
|
|
by_name.setdefault(name, []).append(sha1)
|
|
if suffix and name == "firmware.bin":
|
|
by_suffix.setdefault(suffix, []).append(sha1)
|
|
return {
|
|
"files": files,
|
|
"indexes": {
|
|
"by_name": by_name,
|
|
"by_md5": {entry["md5"]: sha1 for sha1, entry in files.items()},
|
|
"by_sha256": {
|
|
entry["sha256"]: sha1 for sha1, entry in files.items()
|
|
},
|
|
"by_crc32": {},
|
|
"by_path_suffix": by_suffix,
|
|
},
|
|
}
|
|
|
|
def test_hash_identity_wins_over_wrong_destination_hint(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
wrong = root / "wrong" / "firmware.bin"
|
|
correct = root / "correct" / "firmware.bin"
|
|
wrong.parent.mkdir()
|
|
correct.parent.mkdir()
|
|
wrong.write_bytes(b"wrong")
|
|
correct.write_bytes(b"correct")
|
|
db = self._database(
|
|
{"firmware.bin": wrong, "correct-name.bin": correct},
|
|
suffix="Console/USA/firmware.bin",
|
|
)
|
|
expected = hashlib.sha1(b"correct").hexdigest()
|
|
path, status = resolve_local_file(
|
|
{"name": "firmware.bin", "sha1": expected},
|
|
db,
|
|
dest_hint="Console/USA/firmware.bin",
|
|
)
|
|
self.assertEqual(path, str(correct))
|
|
self.assertEqual(status, "sha1_exact")
|
|
|
|
def test_name_cannot_mask_a_declared_hash_mismatch(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
path = Path(directory) / "firmware.bin"
|
|
path.write_bytes(b"wrong")
|
|
db = self._database({"firmware.bin": path})
|
|
resolved, status = resolve_local_file(
|
|
{"name": "firmware.bin", "sha1": "f" * 40}, db
|
|
)
|
|
self.assertEqual(resolved, str(path))
|
|
self.assertEqual(status, "hash_mismatch")
|
|
|
|
|
|
class PackExclusionRegressions(unittest.TestCase):
|
|
def test_slug_platform_core_requirement_uses_the_generated_destination(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
platforms = root / "platforms"
|
|
platforms.mkdir()
|
|
core_payload = root / "extra.bin"
|
|
core_payload.write_bytes(b"mapped core payload")
|
|
core_sha1 = hashlib.sha1(core_payload.read_bytes()).hexdigest()
|
|
core_md5 = hashlib.md5(core_payload.read_bytes()).hexdigest()
|
|
database = {
|
|
"files": {
|
|
core_sha1: {
|
|
"name": "extra.bin",
|
|
"path": str(core_payload),
|
|
"md5": core_md5,
|
|
"size": core_payload.stat().st_size,
|
|
}
|
|
},
|
|
"indexes": {
|
|
"by_name": {"extra.bin": [core_sha1]},
|
|
"by_md5": {core_md5: core_sha1},
|
|
"by_sha256": {},
|
|
"by_crc32": {},
|
|
"by_path_suffix": {},
|
|
},
|
|
}
|
|
config = {
|
|
"platform": "Slug platform",
|
|
"base_destination": "bios",
|
|
"verification_mode": "existence",
|
|
"cores": ["core_a"],
|
|
"systems": {
|
|
"console-a": {
|
|
"files": [
|
|
{
|
|
"name": "base-a.bin",
|
|
"destination": "slug-a/base-a.bin",
|
|
}
|
|
]
|
|
},
|
|
"console-b": {
|
|
"files": [
|
|
{
|
|
"name": "base-b.bin",
|
|
"destination": "slug-b/base-b.bin",
|
|
}
|
|
]
|
|
},
|
|
},
|
|
}
|
|
(platforms / "slug.yml").write_text(
|
|
yaml.safe_dump(config), encoding="utf-8"
|
|
)
|
|
profiles = {
|
|
"core_a": {
|
|
"emulator": "Core A",
|
|
"type": "libretro",
|
|
"systems": ["console-a"],
|
|
"files": [
|
|
{
|
|
"name": "extra.bin",
|
|
"path": "extra.bin",
|
|
"sha1": core_sha1,
|
|
}
|
|
],
|
|
}
|
|
}
|
|
pack = root / "pack.zip"
|
|
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr("slug-a/base-a.bin", b"baseline a")
|
|
archive.writestr("slug-b/base-b.bin", b"baseline b")
|
|
archive.writestr("slug-a/extra.bin", core_payload.read_bytes())
|
|
|
|
result = verify_pack_against_platform(
|
|
str(pack),
|
|
"slug",
|
|
str(platforms),
|
|
db=database,
|
|
emu_profiles=profiles,
|
|
)
|
|
self.assertTrue(result[0], result[3])
|
|
self.assertEqual(result[3], [])
|
|
self.assertEqual(result[6:8], (1, 1))
|
|
|
|
def _mismatch_fixture(self, root: Path, mode: str) -> tuple[dict, Path]:
|
|
"""A platform declaring a hash the only local payload contradicts."""
|
|
platforms = root / "platforms"
|
|
platforms.mkdir()
|
|
payload = root / "firmware.bin"
|
|
payload.write_bytes(b"wrong local variant")
|
|
sha1 = hashlib.sha1(payload.read_bytes()).hexdigest()
|
|
md5 = hashlib.md5(payload.read_bytes()).hexdigest()
|
|
database = {
|
|
"files": {
|
|
sha1: {
|
|
"name": "firmware.bin",
|
|
"path": str(payload),
|
|
"md5": md5,
|
|
"size": payload.stat().st_size,
|
|
}
|
|
},
|
|
"indexes": {
|
|
"by_name": {"firmware.bin": [sha1]},
|
|
"by_md5": {md5: sha1},
|
|
"by_sha256": {},
|
|
"by_crc32": {},
|
|
"by_path_suffix": {},
|
|
},
|
|
}
|
|
config = {
|
|
"platform": f"Platform {mode}",
|
|
"verification_mode": mode,
|
|
"systems": {
|
|
"console": {
|
|
"files": [
|
|
{
|
|
"name": "firmware.bin",
|
|
"destination": "firmware.bin",
|
|
"sha1": "f" * 40,
|
|
}
|
|
]
|
|
}
|
|
},
|
|
}
|
|
(platforms / "plat.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
|
|
return database, platforms
|
|
|
|
def test_hash_platform_accounts_for_an_unsafe_exclusion(self):
|
|
"""A platform that reads the bytes would reject the local payload."""
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
database, platforms = self._mismatch_fixture(root, "md5")
|
|
pack = root / "pack.zip"
|
|
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr("README.txt", "safe subset")
|
|
|
|
result = verify_pack_against_platform(
|
|
str(pack),
|
|
"plat",
|
|
str(platforms),
|
|
db=database,
|
|
emu_profiles={},
|
|
)
|
|
self.assertTrue(result[0], result[3])
|
|
self.assertEqual(result[3], [])
|
|
self.assertEqual(result[8], 1)
|
|
|
|
def test_existence_platform_never_withholds_over_a_declared_hash(self):
|
|
"""RetroArch and friends only look for the filename.
|
|
|
|
An upstream hash the local dump contradicts must not remove a file the
|
|
frontend would have loaded, so its absence stays a conformance error.
|
|
"""
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
database, platforms = self._mismatch_fixture(root, "existence")
|
|
pack = root / "pack.zip"
|
|
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr("README.txt", "no firmware")
|
|
|
|
result = verify_pack_against_platform(
|
|
str(pack),
|
|
"plat",
|
|
str(platforms),
|
|
db=database,
|
|
emu_profiles={},
|
|
)
|
|
self.assertFalse(result[0])
|
|
self.assertTrue(any("baseline missing" in e for e in result[3]), result[3])
|
|
self.assertEqual(result[8], 0)
|
|
|
|
def test_unexplained_missing_file_still_fails_conformance(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
platforms = root / "platforms"
|
|
platforms.mkdir()
|
|
config = {
|
|
"platform": "Missing",
|
|
"verification_mode": "existence",
|
|
"systems": {
|
|
"console": {
|
|
"files": [
|
|
{"name": "absent.bin", "destination": "absent.bin"}
|
|
]
|
|
}
|
|
},
|
|
}
|
|
(platforms / "missing.yml").write_text(
|
|
yaml.safe_dump(config), encoding="utf-8"
|
|
)
|
|
pack = root / "pack.zip"
|
|
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr("README.txt", "incomplete")
|
|
database = {
|
|
"files": {},
|
|
"indexes": {
|
|
"by_name": {},
|
|
"by_md5": {},
|
|
"by_sha256": {},
|
|
"by_crc32": {},
|
|
"by_path_suffix": {},
|
|
},
|
|
}
|
|
|
|
result = verify_pack_against_platform(
|
|
str(pack),
|
|
"missing",
|
|
str(platforms),
|
|
db=database,
|
|
emu_profiles={},
|
|
)
|
|
self.assertFalse(result[0])
|
|
self.assertTrue(any("baseline missing" in error for error in result[3]))
|
|
|
|
|
|
class RegionRegressions(unittest.TestCase):
|
|
def test_tagged_and_untagged_same_path_is_preserved(self):
|
|
profiles = {
|
|
"tagged": {
|
|
"type": "libretro",
|
|
"files": [
|
|
{"name": "bios.bin", "path": "sys/bios.bin", "region": ["japan"]}
|
|
],
|
|
},
|
|
"untagged": {
|
|
"type": "libretro",
|
|
"files": [{"name": "bios.bin", "path": "sys/bios.bin"}],
|
|
},
|
|
}
|
|
index = region.build_region_index(profiles)
|
|
self.assertEqual(region.lookup_regions(index, "sys/bios.bin", "bios.bin"), set())
|
|
drops = region.resolve_region_drops(
|
|
{"system": [("sys/bios.bin", "bios.bin")]},
|
|
index,
|
|
["north-america"],
|
|
)
|
|
self.assertEqual(drops, set())
|
|
|
|
def test_world_candidate_beats_unmatched_regional_fallback(self):
|
|
profiles = {
|
|
"core": {
|
|
"type": "libretro",
|
|
"files": [
|
|
{"name": "ntsc.bin", "region": ["world"]},
|
|
{"name": "pal.bin", "region": ["europe"]},
|
|
],
|
|
}
|
|
}
|
|
index = region.build_region_index(profiles)
|
|
groups = {"system": [("ntsc.bin", "ntsc.bin"), ("pal.bin", "pal.bin")]}
|
|
self.assertEqual(
|
|
region.resolve_region_drops(groups, index, ["north-america"]),
|
|
{"pal.bin"},
|
|
)
|
|
self.assertEqual(region.resolve_region_drops(groups, index, ["europe"]), set())
|
|
|
|
def test_multi_system_emulator_uses_separate_region_groups(self):
|
|
profile = {"systems": ["odyssey2", "videopac"]}
|
|
north_america = _emulator_region_group(
|
|
"o2em", profile, {"name": "o2rom.bin", "system": "odyssey2"}
|
|
)
|
|
europe = _emulator_region_group(
|
|
"o2em", profile, {"name": "c52.bin", "system": "videopac"}
|
|
)
|
|
self.assertNotEqual(north_america, europe)
|
|
|
|
def test_region_audit_accepts_list_valued_md5(self):
|
|
sha1 = "a" * 40
|
|
md5 = "b" * 32
|
|
db = {
|
|
"files": {sha1: {}},
|
|
"indexes": {"by_md5": {md5: sha1}, "by_name": {}},
|
|
}
|
|
self.assertEqual(
|
|
region_audit.resolve_sha1({"name": "bios.bin", "md5": [md5]}, db),
|
|
sha1,
|
|
)
|
|
|
|
|
|
class ArchiveSecurityRegressions(unittest.TestCase):
|
|
def _zip_path(self, directory: Path, name: str = "archive.zip") -> Path:
|
|
return directory / name
|
|
|
|
def test_member_count_limit_is_enforced(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
with zipfile.ZipFile(archive, "w") as handle:
|
|
handle.writestr("one.bin", b"1")
|
|
handle.writestr("two.bin", b"2")
|
|
with self.assertRaisesRegex(ValueError, "members"):
|
|
safe_extract_zip(str(archive), str(root / "out"), max_members=1)
|
|
|
|
def test_symlink_member_is_rejected(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
link = zipfile.ZipInfo("link")
|
|
link.create_system = 3
|
|
link.external_attr = (stat.S_IFLNK | 0o777) << 16
|
|
with zipfile.ZipFile(archive, "w") as handle:
|
|
handle.writestr(link, "target")
|
|
with self.assertRaisesRegex(ValueError, "link or special"):
|
|
safe_extract_zip(str(archive), str(root / "out"))
|
|
|
|
def test_windows_separator_is_a_path_not_a_rejection(self):
|
|
"""Archives written on Windows store a backslash separator.
|
|
|
|
download.py feeds third-party archives to this function, so a legal
|
|
Windows path must extract into a subdirectory instead of failing.
|
|
"""
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
with zipfile.ZipFile(archive, "w") as handle:
|
|
handle.writestr("sub\\rom.bin", b"payload")
|
|
out = root / "out"
|
|
safe_extract_zip(str(archive), str(out))
|
|
self.assertEqual((out / "sub" / "rom.bin").read_bytes(), b"payload")
|
|
|
|
def test_windows_separator_cannot_smuggle_traversal(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
with zipfile.ZipFile(archive, "w") as handle:
|
|
handle.writestr("..\\escaped.bin", b"payload")
|
|
with self.assertRaisesRegex(ValueError, "traversal"):
|
|
safe_extract_zip(str(archive), str(root / "out"))
|
|
|
|
def test_high_ratio_method_is_bounded_by_size_not_ratio(self):
|
|
"""bzip2 legitimately exceeds the DEFLATE ceiling."""
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
with zipfile.ZipFile(archive, "w", zipfile.ZIP_BZIP2) as handle:
|
|
handle.writestr("zeros.bin", bytes(4_000_000))
|
|
out = root / "out"
|
|
safe_extract_zip(str(archive), str(out))
|
|
self.assertEqual((out / "zeros.bin").stat().st_size, 4_000_000)
|
|
|
|
def test_compression_ratio_limit_is_enforced(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
archive = self._zip_path(root)
|
|
with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as handle:
|
|
handle.writestr("zeros.bin", bytes(16_384))
|
|
with self.assertRaisesRegex(ValueError, "compression ratio"):
|
|
safe_extract_zip(
|
|
str(archive), str(root / "out"), max_compression_ratio=2
|
|
)
|
|
|
|
|
|
class NoMd5IsNotNothingChecked(unittest.TestCase):
|
|
"""A declared hash that is not an md5 still contradicts.
|
|
|
|
verify_entry_md5 returned OK the moment the entry declared no md5, so an
|
|
entry whose sha256 the local file contradicts read as covered while the
|
|
builder was already excluding it. RetroDECK's dsifirmware.bin declares a
|
|
sha256 and no md5.
|
|
"""
|
|
|
|
def setUp(self):
|
|
import verify
|
|
|
|
self.verify = verify
|
|
|
|
def _entry(self):
|
|
return {"name": "dsifirmware.bin", "sha256": "b" * 64}
|
|
|
|
def test_a_contradicted_sha256_is_not_reported_ok(self):
|
|
result = self.verify.verify_entry_md5(
|
|
self._entry(), "bios/whatever.bin", "hash_mismatch"
|
|
)
|
|
self.assertNotEqual(result["status"], self.verify.Status.OK)
|
|
self.assertIn("contradicted", result.get("reason", ""))
|
|
|
|
def test_a_clean_resolution_is_still_ok(self):
|
|
result = self.verify.verify_entry_md5(
|
|
self._entry(), "bios/whatever.bin", "sha256_exact"
|
|
)
|
|
self.assertEqual(result["status"], self.verify.Status.OK)
|
|
|
|
|
|
class InstallerBoundaryRegressions(unittest.TestCase):
|
|
def _manifest(self, dest: str) -> dict:
|
|
return {
|
|
"manifest_version": 2,
|
|
"platform": "retroarch",
|
|
"files": [
|
|
{
|
|
"dest": dest,
|
|
"size": 1,
|
|
"sha1": "a" * 40,
|
|
"sha256": "b" * 64,
|
|
"repo_path": "bios/test.bin",
|
|
"cores": None,
|
|
}
|
|
],
|
|
"standalone_copies": [],
|
|
}
|
|
|
|
def test_manifest_destination_traversal_is_rejected(self):
|
|
with self.assertRaisesRegex(ValueError, "unsafe"):
|
|
install._validate_manifest(self._manifest("../escape"), "retroarch")
|
|
|
|
def test_manifest_repo_source_is_confined_to_bios(self):
|
|
manifest = self._manifest("safe.bin")
|
|
manifest["files"][0]["repo_path"] = "scripts/pipeline.py"
|
|
with self.assertRaisesRegex(ValueError, "outside bios"):
|
|
install._validate_manifest(manifest, "retroarch")
|
|
|
|
def test_omitted_destination_cannot_overlap_a_download(self):
|
|
manifest = self._manifest("safe.bin")
|
|
manifest["omitted_files"] = [
|
|
{
|
|
"dest": "safe.bin",
|
|
"name": "safe.bin",
|
|
"system": "console",
|
|
"required": True,
|
|
"reason": "hash_mismatch",
|
|
"cores": None,
|
|
}
|
|
]
|
|
manifest["total_omitted"] = 1
|
|
with self.assertRaisesRegex(ValueError, "conflicting omitted"):
|
|
install._validate_manifest(manifest, "retroarch")
|
|
|
|
def test_omitted_destination_traversal_is_rejected(self):
|
|
manifest = self._manifest("safe.bin")
|
|
manifest["omitted_files"] = [
|
|
{
|
|
"dest": "../unsafe.bin",
|
|
"name": "unsafe.bin",
|
|
"system": "console",
|
|
"required": True,
|
|
"reason": "hash_mismatch",
|
|
"cores": None,
|
|
}
|
|
]
|
|
manifest["total_omitted"] = 1
|
|
with self.assertRaisesRegex(ValueError, "unsafe"):
|
|
install._validate_manifest(manifest, "retroarch")
|
|
|
|
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
|
def test_target_schema_accepts_the_null_the_generator_emits(self):
|
|
"""The schema and generate_target_manifests must agree on null.
|
|
|
|
A target with no core list is written as null; a schema that rejects
|
|
it turns a valid manifest into a CI failure.
|
|
"""
|
|
from jsonschema import Draft202012Validator
|
|
|
|
schema = json.loads(
|
|
(ROOT / "schemas" / "target-manifest.schema.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
validator = Draft202012Validator(schema)
|
|
document = {"windows": None, "switch": ["a5200"]}
|
|
self.assertEqual(list(validator.iter_errors(document)), [])
|
|
|
|
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
|
def test_pack_manifests_are_read_from_inside_the_archive(self):
|
|
"""generate_pack writes manifest.json into the ZIP, not beside it.
|
|
|
|
A filesystem glob over dist/ matches nothing and reports success
|
|
without validating a single document.
|
|
"""
|
|
import validate_schemas
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
dist = Path(directory)
|
|
broken = {
|
|
"schema_version": 1,
|
|
"version": 1,
|
|
"generator": "retrobios generate_pack.py",
|
|
"generated": "2026-08-10T00:00:00Z",
|
|
"files": [{"path": "a.bin", "sha1": "nope", "md5": "nope",
|
|
"size": 1, "status": "verified", "name": "a.bin"}],
|
|
"summary": {"total_files": 1, "verified": 1, "untracked": 0,
|
|
"errors": 0},
|
|
"errors": [],
|
|
}
|
|
with zipfile.ZipFile(dist / "Pack.zip", "w") as archive:
|
|
archive.writestr("manifest.json", json.dumps(broken))
|
|
|
|
errors = validate_schemas._validate_pack_manifests(dist)
|
|
self.assertTrue(errors, "an invalid in-archive manifest must be reported")
|
|
self.assertTrue(any("sha1" in message for message in errors), errors)
|
|
|
|
def test_null_core_list_keeps_the_other_targets(self):
|
|
"""A target without a core inventory must not void the manifest.
|
|
|
|
generate_target_manifests emits null for a target that publishes no
|
|
core list; rejecting the document would silently disable --target for
|
|
every target on that platform.
|
|
"""
|
|
normalized = install._validate_targets({"windows": None, "switch": ["a5200"]})
|
|
self.assertIsNone(normalized["windows"]["cores"])
|
|
self.assertEqual(normalized["switch"]["cores"], ["a5200"])
|
|
|
|
def test_legacy_target_lists_are_normalized(self):
|
|
self.assertEqual(
|
|
install._validate_targets({"rpi4": ["core-a", "core-b"]}),
|
|
{"rpi4": {"cores": ["core-a", "core-b"]}},
|
|
)
|
|
|
|
|
|
class TargetManifestRegressions(unittest.TestCase):
|
|
def test_yaml_scalar_core_is_rejected_instead_of_leaking_to_json(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
source = root / "source"
|
|
output = root / "output"
|
|
source.mkdir()
|
|
(source / "platform.yml").write_text(
|
|
"targets:\n device:\n cores: [81, valid-core]\n",
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "non-empty strings"):
|
|
generate_target_manifests(str(source), str(output))
|
|
|
|
|
|
class CheckoutCompletenessRegressions(unittest.TestCase):
|
|
"""Coverage is resolved against the disk, so the checkout must be whole.
|
|
|
|
Files over 50 MB and the data directory caches are gitignored. A job that
|
|
regenerates the README or the site without restoring them counts those
|
|
files as missing and publishes a coverage the collection does not have.
|
|
"""
|
|
|
|
def _steps(self, workflow: str, job: str) -> list[dict]:
|
|
data = yaml.safe_load((ROOT / ".github" / "workflows" / workflow).read_text())
|
|
return data["jobs"][job]["steps"]
|
|
|
|
def _index(self, steps: list[dict], needle: str) -> int:
|
|
for position, step in enumerate(steps):
|
|
if needle in step.get("name", "") or needle in str(step.get("run", "")):
|
|
return position
|
|
self.fail(f"no step matching {needle!r}")
|
|
|
|
def test_site_deploy_completes_the_checkout_before_generating(self):
|
|
steps = self._steps("deploy-site.yml", "build")
|
|
generate = self._index(steps, "Generate site")
|
|
self.assertLess(self._index(steps, "restore_large_files.py"), generate)
|
|
self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate)
|
|
|
|
def test_restore_matches_assets_by_content_not_by_name(self):
|
|
from scripts.restore_large_files import restore
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
cache = root / "cache"
|
|
cache.mkdir()
|
|
payload = b"firmware bytes"
|
|
(cache / "renamed-asset.bin").write_bytes(payload)
|
|
sha1 = hashlib.sha1(payload).hexdigest()
|
|
(root / ".gitignore").write_text("bios/Sony/big.pup\n", encoding="utf-8")
|
|
(root / "database.json").write_text(
|
|
json.dumps({"files": {sha1: {"path": "bios/Sony/big.pup"}}}),
|
|
encoding="utf-8",
|
|
)
|
|
cwd = os.getcwd()
|
|
os.chdir(root)
|
|
try:
|
|
restored, unsatisfied = restore(
|
|
str(cache), "database.json", ".gitignore"
|
|
)
|
|
self.assertEqual((restored, unsatisfied), (1, []))
|
|
self.assertEqual((root / "bios/Sony/big.pup").read_bytes(), payload)
|
|
# A path already in the checkout is never overwritten.
|
|
restored, unsatisfied = restore(
|
|
str(cache), "database.json", ".gitignore"
|
|
)
|
|
self.assertEqual((restored, unsatisfied), (0, []))
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def test_restore_leaves_tracked_paths_alone(self):
|
|
from scripts.restore_large_files import restore
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
cache = root / "cache"
|
|
cache.mkdir()
|
|
payload = b"tracked bytes"
|
|
(cache / "asset.bin").write_bytes(payload)
|
|
sha1 = hashlib.sha1(payload).hexdigest()
|
|
(root / ".gitignore").write_text("bios/other.bin\n", encoding="utf-8")
|
|
(root / "database.json").write_text(
|
|
json.dumps({"files": {sha1: {"path": "bios/tracked.bin"}}}),
|
|
encoding="utf-8",
|
|
)
|
|
cwd = os.getcwd()
|
|
os.chdir(root)
|
|
try:
|
|
restored, unsatisfied = restore(
|
|
str(cache), "database.json", ".gitignore"
|
|
)
|
|
self.assertEqual((restored, unsatisfied), (0, []))
|
|
self.assertFalse((root / "bios/tracked.bin").exists())
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
|
|
class EveryManifestEntryIsFetchable(unittest.TestCase):
|
|
"""An install manifest may not list a file the installer cannot get.
|
|
|
|
install.py fetches a file either from its repo_path or from a release
|
|
asset. An entry carrying neither is a line in the download list that can
|
|
only ever fail. It happened when resolution landed on a file the database
|
|
does not index, so the hash was computed but the repo lookup came back
|
|
empty and the entry shipped anyway.
|
|
"""
|
|
|
|
def test_committed_manifests_all_have_a_source(self):
|
|
manifests = sorted((ROOT / "install").glob("*.json"))
|
|
self.assertTrue(manifests, "no install manifests to check")
|
|
for path in manifests:
|
|
with self.subTest(manifest=path.name):
|
|
data = json.loads(path.read_text())
|
|
orphans = [
|
|
entry["dest"]
|
|
for entry in data.get("files", [])
|
|
if not entry.get("repo_path") and not entry.get("release_asset")
|
|
]
|
|
self.assertEqual(
|
|
orphans, [], f"{path.name} lists unfetchable files: {orphans[:3]}"
|
|
)
|
|
|
|
def test_an_unresolvable_file_is_recorded_as_omitted(self):
|
|
"""The reason must be one install.py knows how to report."""
|
|
allowed = {"hash_mismatch", "not_found", "external", "user_provided"}
|
|
for path in sorted((ROOT / "install").glob("*.json")):
|
|
with self.subTest(manifest=path.name):
|
|
data = json.loads(path.read_text())
|
|
for entry in data.get("omitted_files", []):
|
|
self.assertIn(entry.get("reason"), allowed)
|
|
|
|
def _manifests(self) -> list[Path]:
|
|
paths = sorted((ROOT / "install").glob("*.json"))
|
|
paths += sorted((ROOT / "install" / "targets").glob("*.json"))
|
|
self.assertTrue(paths, "no install manifests to check")
|
|
return paths
|
|
|
|
def test_repo_paths_name_the_file_the_database_holds(self):
|
|
"""A manifest written before a file moved sends the installer to 404.
|
|
|
|
database.json is regenerated whenever a file is refiled; the manifests
|
|
are a separate artefact and were left behind once (the two DS
|
|
firmware dumps of the 7 September refile). The installer fetches
|
|
repo_path from raw.githubusercontent.com, so a stale one is a
|
|
download that can never succeed.
|
|
"""
|
|
database = ROOT / "database.json"
|
|
if not database.is_file():
|
|
self.skipTest("database.json not generated")
|
|
files = json.loads(database.read_text(encoding="utf-8"))["files"]
|
|
for path in self._manifests():
|
|
with self.subTest(manifest=path.name):
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
stale = [
|
|
(entry["dest"], entry["repo_path"], files.get(entry["sha1"], {}).get("path"))
|
|
for entry in data.get("files", [])
|
|
if entry.get("repo_path")
|
|
and files.get(entry.get("sha1", ""), {}).get("path") != entry["repo_path"]
|
|
]
|
|
self.assertEqual(stale, [], f"{path.name} names moved files: {stale[:3]}")
|
|
|
|
def test_release_entries_are_exactly_the_files_git_does_not_hold(self):
|
|
"""The installer has two sources, and the manifest must pick the right one.
|
|
|
|
A committed file is served by the repository; a gitignored one by
|
|
the large-files release. Marking a committed file as a release asset
|
|
sends the installer to an asset nobody uploaded (sdlpal/5.avi, 83 MB
|
|
in git, was announced as release asset `5.avi`). Marking a gitignored
|
|
file as a repo file sends it to a raw URL that has no content.
|
|
"""
|
|
tracked = set(
|
|
subprocess.run(
|
|
["git", "ls-files", "-z", "--", "bios"],
|
|
capture_output=True, text=True, cwd=ROOT, check=True,
|
|
).stdout.split("\0")
|
|
)
|
|
if not tracked:
|
|
self.skipTest("not a git checkout")
|
|
for path in self._manifests():
|
|
with self.subTest(manifest=path.name):
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
wrong = []
|
|
for entry in data.get("files", []):
|
|
repo_path = entry.get("repo_path")
|
|
if not repo_path:
|
|
continue
|
|
committed = repo_path in tracked
|
|
if bool(entry.get("release_asset")) == committed:
|
|
wrong.append((entry["dest"], repo_path, "committed" if committed else "gitignored"))
|
|
self.assertEqual(wrong, [], f"{path.name} picks the wrong source: {wrong[:3]}")
|
|
|
|
|
|
class ReleaseAssetCriterion(unittest.TestCase):
|
|
"""What decides whether the installer fetches from the release is git, not size.
|
|
|
|
Files over 50 MB are supposed to be gitignored and uploaded to the
|
|
large-files release, but four committed files break that rule and one of
|
|
them reached a manifest as a release asset that does not exist. The
|
|
truthful question is whether the repository serves the bytes, and the
|
|
ledger of what it does not serve is .gitignore.
|
|
"""
|
|
|
|
def _repo(self, directory: str, ignored: str) -> str:
|
|
Path(directory, ".gitignore").write_text(ignored + "\n", encoding="utf-8")
|
|
return directory
|
|
|
|
def test_a_large_committed_file_is_served_by_the_repository(self):
|
|
from scripts import generate_pack
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = self._repo(directory, "bios/elsewhere.bin")
|
|
big = Path(root, "bios", "big.avi")
|
|
big.parent.mkdir()
|
|
with big.open("wb") as handle:
|
|
handle.truncate(60_000_000)
|
|
generate_pack._GITIGNORE_ENTRIES = None
|
|
try:
|
|
self.assertFalse(generate_pack._is_release_asset(str(big), root))
|
|
finally:
|
|
generate_pack._GITIGNORE_ENTRIES = None
|
|
|
|
def test_a_gitignored_file_is_served_by_the_release_whatever_its_size(self):
|
|
from scripts import generate_pack
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = self._repo(directory, "bios/small.zip")
|
|
small = Path(root, "bios", "small.zip")
|
|
small.parent.mkdir()
|
|
small.write_bytes(b"PK")
|
|
generate_pack._GITIGNORE_ENTRIES = None
|
|
try:
|
|
self.assertTrue(generate_pack._is_release_asset(str(small), root))
|
|
finally:
|
|
generate_pack._GITIGNORE_ENTRIES = None
|
|
|
|
|
|
class ReleaseAssetsMatchTheCollection(unittest.TestCase):
|
|
"""The bytes the release serves must be the bytes the manifests describe.
|
|
|
|
install.py checks Content-Length against the manifest size before it
|
|
downloads, and the manifest size is that of the local copy. Both FBNeo
|
|
sample archives were rebuilt locally after their upload and never
|
|
re-uploaded: same members, different container, every install of them
|
|
failed. The comparison is offline here on a fixture; the script runs it
|
|
against the live release.
|
|
"""
|
|
|
|
def test_missing_and_resized_assets_are_reported_and_nothing_else(self):
|
|
from scripts import check_release_assets
|
|
|
|
expected = {
|
|
"bios/Other/fbneo/fbneo/samples/twotiger.zip": 154985212,
|
|
"bios/Arcade/MAME/MAME 0.174 Arcade XML.dat": 53677408,
|
|
"bios/Other/sdlpal/sdlpal/5.avi": 82833972,
|
|
}
|
|
assets = {
|
|
"twotiger.zip": 154888877,
|
|
"MAME.0.174.Arcade.XML.dat": 53677408,
|
|
"unrelated.zip": 1,
|
|
}
|
|
findings = check_release_assets.compare(expected, assets)
|
|
self.assertEqual(
|
|
findings,
|
|
[
|
|
("missing", "bios/Other/sdlpal/sdlpal/5.avi", 82833972, None),
|
|
("size", "bios/Other/fbneo/fbneo/samples/twotiger.zip", 154985212, 154888877),
|
|
],
|
|
)
|
|
|
|
def test_expected_assets_are_the_gitignored_database_entries(self):
|
|
from scripts import check_release_assets
|
|
|
|
db = {
|
|
"files": {
|
|
"a" * 40: {"path": "bios/Other/x/big.zip", "size": 10},
|
|
"b" * 40: {"path": "bios/Other/x/small.bin", "size": 2},
|
|
}
|
|
}
|
|
gitignore = "# large\nbios/Other/x/big.zip\ntmp/\n"
|
|
self.assertEqual(
|
|
check_release_assets.expected_assets(db, gitignore),
|
|
{"bios/Other/x/big.zip": 10},
|
|
)
|
|
|
|
def test_release_notes_are_rendered_from_the_collection(self):
|
|
"""The release page is written by hand and drifts; it is rendered instead.
|
|
|
|
Three assets had no row, three rows carried the SHA1 of an asset
|
|
that had since been replaced, and the footer counted 41 files for 44.
|
|
Hand-written descriptions survive by name; a new row takes its
|
|
description from the profile that declares the file; an asset the
|
|
collection does not index is listed apart, with its size only.
|
|
"""
|
|
from scripts import check_release_assets
|
|
|
|
db = {"files": {
|
|
"a" * 40: {"path": "bios/Nintendo/Wii/nand.bin", "size": 553649152},
|
|
"b" * 40: {"path": "bios/Other/fbneo/fbneo/samples/twotiger.zip", "size": 154985212},
|
|
"c" * 40: {"path": "bios/Other/x/small.bin", "size": 2},
|
|
}}
|
|
gitignore = "bios/Nintendo/Wii/nand.bin\nbios/Other/fbneo/fbneo/samples/twotiger.zip\n"
|
|
assets = {"nand.bin": 553649152, "twotiger.zip": 154985212,
|
|
"sdlpal-data.zip": 140773720, "whoopee.zip": 222665810}
|
|
previous = (
|
|
"## Arcade\n\n| File | Description | Size | SHA1 |\n|---|---|---|---|\n"
|
|
"| [twotiger.zip](u) | Two Tigers samples | 148 MB | `" + "0" * 40 + "` |\n"
|
|
"| [whoopee.zip](u) | Toaplan Whoopee samples | 212 MB | `" + "9" * 40 + "` |\n"
|
|
)
|
|
descriptions = {"nand.bin": "BootMii NAND backup"}
|
|
bundles = {"sdlpal-data.zip": "SDLPAL Chinese Paladin game data (.mkf archives)"}
|
|
body = check_release_assets.render_notes(
|
|
db, gitignore, assets, previous, descriptions, bundles, {"sdlpal-data.zip": "d" * 40}
|
|
)
|
|
self.assertIn("| [twotiger.zip](https://github.com/Abdess/retrobios/releases/download/large-files/twotiger.zip) | Two Tigers samples | 148 MB | `" + "b" * 40 + "` |", body)
|
|
self.assertIn("| [nand.bin](https://github.com/Abdess/retrobios/releases/download/large-files/nand.bin) | BootMii NAND backup | 528 MB | `" + "a" * 40 + "` |", body)
|
|
self.assertIn("| [sdlpal-data.zip](https://github.com/Abdess/retrobios/releases/download/large-files/sdlpal-data.zip) | SDLPAL Chinese Paladin game data (.mkf archives) | 134 MB | `" + "d" * 40 + "` |", body)
|
|
self.assertIn("## Not indexed", body)
|
|
self.assertIn("| [whoopee.zip](https://github.com/Abdess/retrobios/releases/download/large-files/whoopee.zip) | Toaplan Whoopee samples | 212 MB |", body)
|
|
self.assertNotIn("9" * 40, body)
|
|
self.assertNotIn("small.bin", body)
|
|
self.assertIn("4 files, 1.0 GB total", body)
|
|
self.assertEqual(body.index("## Console firmware"), min(
|
|
body.index(h) for h in ("## Console firmware", "## Arcade", "## Not indexed")
|
|
))
|
|
|
|
def test_stale_notes_are_a_finding(self):
|
|
from scripts import check_release_assets
|
|
|
|
findings = check_release_assets.compare({}, {}, notes_current=False)
|
|
self.assertEqual(findings, [("notes", "release description", 0, None)])
|
|
|
|
def test_pipeline_runs_the_check_online_and_marks_it_skipped_offline(self):
|
|
source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
|
|
self.assertIn("check_release_assets.py", source)
|
|
self.assertIn('results["check_release_assets"] = SKIPPED', source)
|
|
|
|
|
|
class PreservedBytesAreNeverNormalised(unittest.TestCase):
|
|
"""git must not rewrite a preserved file's line endings.
|
|
|
|
SHA1 is the primary key of this collection. Git for Windows sets
|
|
core.autocrlf=true by default, so without an attribute saying otherwise a
|
|
clone there rewrites every file git guesses is text -- shaders, .ini,
|
|
.txt and .dat assets under bios/ -- and each one arrives with a different
|
|
hash from the one published here.
|
|
"""
|
|
|
|
def _attr(self, path: str) -> str:
|
|
import subprocess
|
|
|
|
out = subprocess.run(
|
|
["git", "check-attr", "text", "--", path],
|
|
capture_output=True, text=True, cwd=ROOT,
|
|
).stdout
|
|
return out.rsplit(":", 1)[-1].strip()
|
|
|
|
def test_gitattributes_exists(self):
|
|
self.assertTrue(
|
|
(ROOT / ".gitattributes").is_file(),
|
|
"without it git guesses, and guesses wrong on Windows",
|
|
)
|
|
|
|
def test_collection_paths_are_exempt_from_normalisation(self):
|
|
for path in (
|
|
"bios/Sony/PlayStation/scph5501.bin",
|
|
"bios/Other/j2me-loader/color.fsh",
|
|
"data/anything.txt",
|
|
"data/dolphin-sys/config.json",
|
|
):
|
|
with self.subTest(path=path):
|
|
self.assertEqual(self._attr(path), "unset", f"{path} may be rewritten")
|
|
|
|
def test_generated_artefacts_stay_lf(self):
|
|
for path in (
|
|
"database.json",
|
|
"scripts/dedup.py",
|
|
"README.md",
|
|
"provenance/redump.json",
|
|
):
|
|
with self.subTest(path=path):
|
|
self.assertEqual(self._attr(path), "set")
|
|
|
|
|
|
class FreshnessGuardMechanics(unittest.TestCase):
|
|
"""write_if_changed is what makes `git diff --exit-code` a real check.
|
|
|
|
deploy-site.yml regenerates README.md and CONTRIBUTING.md and then fails
|
|
if git sees a change. That is only a staleness check because a run which
|
|
moves nothing but the clock leaves the file untouched; if the comparison
|
|
missed a timestamp form, the guard would fail on every run and stop
|
|
meaning anything.
|
|
"""
|
|
|
|
def setUp(self):
|
|
from common import write_if_changed
|
|
|
|
self.write_if_changed = write_if_changed
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.path = os.path.join(self._tmp.name, "page.md")
|
|
|
|
def tearDown(self):
|
|
self._tmp.cleanup()
|
|
|
|
def test_a_new_file_is_written(self):
|
|
self.assertTrue(self.write_if_changed(self.path, "body\n"))
|
|
with open(self.path) as handle:
|
|
self.assertEqual(handle.read(), "body\n")
|
|
|
|
def test_identical_content_is_not_rewritten(self):
|
|
self.write_if_changed(self.path, "body\n")
|
|
self.assertFalse(self.write_if_changed(self.path, "body\n"))
|
|
|
|
def test_real_change_is_written(self):
|
|
self.write_if_changed(self.path, "body\n")
|
|
self.assertTrue(self.write_if_changed(self.path, "other\n"))
|
|
|
|
def test_every_timestamp_form_is_ignored_on_its_own(self):
|
|
forms = [
|
|
'{{"generated_at": "{}"}}',
|
|
'{{"imported_at": "{}"}}',
|
|
"*Auto-generated on {}*",
|
|
"*Generated on {}*",
|
|
'<div class="rb-timestamp">Generated on {}.</div>',
|
|
]
|
|
for form in forms:
|
|
with self.subTest(form=form):
|
|
first = form.format("2026-01-01T00:00:00Z")
|
|
second = form.format("2026-09-09T09:09:09Z")
|
|
self.write_if_changed(self.path, first)
|
|
self.assertFalse(
|
|
self.write_if_changed(self.path, second),
|
|
f"a clock-only change rewrote the file for {form!r}",
|
|
)
|
|
|
|
def test_a_change_beside_a_moving_timestamp_is_still_written(self):
|
|
self.write_if_changed(self.path, "count: 1\n*Generated on A*\n")
|
|
self.assertTrue(
|
|
self.write_if_changed(self.path, "count: 2\n*Generated on B*\n")
|
|
)
|
|
|
|
def test_the_written_file_keeps_the_new_timestamp_when_content_changed(self):
|
|
self.write_if_changed(self.path, "count: 1\n*Generated on A*\n")
|
|
self.write_if_changed(self.path, "count: 2\n*Generated on B*\n")
|
|
with open(self.path) as handle:
|
|
self.assertIn("Generated on B", handle.read())
|
|
|
|
|
|
|
|
class PipelineReportsWhatItDid(unittest.TestCase):
|
|
"""A step that did not run must not be summarised as OK.
|
|
|
|
--offline skips the network steps and --with-export gates three more.
|
|
Reporting them as OK claims work nobody did, and a run whose export
|
|
never happened read exactly like one where it had.
|
|
"""
|
|
|
|
def test_a_skipped_step_is_not_reported_as_done(self):
|
|
import pipeline
|
|
|
|
self.assertTrue(bool(pipeline.SKIPPED), "a skip must not fail the run")
|
|
self.assertEqual(repr(pipeline.SKIPPED), "SKIPPED")
|
|
self.assertIsNot(pipeline.SKIPPED, True)
|
|
|
|
def test_every_skip_branch_uses_the_sentinel(self):
|
|
source = (ROOT / "scripts" / "pipeline.py").read_text()
|
|
stale = re.findall(r'results\["(\w+)"\] = True', source)
|
|
self.assertEqual(stale, [], f"steps still claiming OK when skipped: {stale}")
|
|
|
|
class DeclaredDependenciesMatchTheDocumentedWorkflows(unittest.TestCase):
|
|
"""What the workflows install has to be what the project declares.
|
|
|
|
mkdocs-material and pymdown-extensions were hard requirements of the site
|
|
build in CI and in the release guide while pyproject declared neither, so
|
|
a contributor following either had to read the workflow to find out.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _pyproject() -> dict:
|
|
import tomllib
|
|
|
|
with (ROOT / "pyproject.toml").open("rb") as handle:
|
|
return tomllib.load(handle)
|
|
|
|
def test_every_package_ci_installs_is_declared(self):
|
|
extras = self._pyproject()["project"]["optional-dependencies"]
|
|
declared = {
|
|
name.split(">")[0].split("=")[0].split("<")[0].strip('"')
|
|
for group in extras.values()
|
|
for name in group
|
|
}
|
|
declared |= set(self._pyproject()["project"]["dependencies"])
|
|
installed: set[str] = set()
|
|
for workflow in ("validate.yml", "deploy-site.yml"):
|
|
body = (ROOT / ".github" / "workflows" / workflow).read_text(
|
|
encoding="utf-8"
|
|
)
|
|
for line in body.splitlines():
|
|
if "pip install" not in line:
|
|
continue
|
|
for token in line.split("pip install", 1)[1].split():
|
|
name = token.strip('"').split(">")[0].split("=")[0]
|
|
name = name.split("<")[0].strip()
|
|
if name:
|
|
installed.add(name)
|
|
self.assertEqual(
|
|
installed - declared,
|
|
set(),
|
|
"CI installs packages pyproject does not declare",
|
|
)
|
|
|
|
|
|
class ACheckThatCannotAnswerDoesNotPass(unittest.TestCase):
|
|
"""Exiting zero says the question was answered and the answer was yes.
|
|
|
|
Four scripts said that without answering: a refresh that reached no
|
|
remote, a freshness check whose upstream was unreachable, a restore whose
|
|
cache could not supply a declared path, and a pack verification asked
|
|
about one platform whose pack was not there.
|
|
"""
|
|
|
|
def test_naming_a_platform_with_no_pack_is_not_a_pass(self):
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
named = subprocess.run(
|
|
[sys.executable, "scripts/generate_pack.py", "--platform",
|
|
"retroarch", "--verify-packs", "--output-dir", directory],
|
|
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
|
|
)
|
|
self.assertNotEqual(
|
|
named.returncode, 0,
|
|
"a named platform with no pack reported success:\n"
|
|
+ named.stdout + named.stderr,
|
|
)
|
|
# --all is a sweep: a platform nobody built is out of scope.
|
|
swept = subprocess.run(
|
|
[sys.executable, "scripts/generate_pack.py", "--all",
|
|
"--verify-packs", "--output-dir", directory],
|
|
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
|
|
)
|
|
self.assertEqual(swept.returncode, 0, swept.stdout + swept.stderr)
|
|
|
|
def test_an_unsatisfiable_declared_path_is_reported(self):
|
|
from scripts.restore_large_files import restore
|
|
|
|
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
|
root = Path(directory)
|
|
cache = root / "cache"
|
|
cache.mkdir()
|
|
(root / ".gitignore").write_text("bios/absent.bin\n", encoding="utf-8")
|
|
(root / "database.json").write_text(
|
|
json.dumps({"files": {"a" * 40: {"path": "bios/absent.bin"}}}),
|
|
encoding="utf-8",
|
|
)
|
|
cwd = os.getcwd()
|
|
os.chdir(root)
|
|
try:
|
|
restored, unsatisfied = restore(
|
|
str(cache), "database.json", ".gitignore"
|
|
)
|
|
finally:
|
|
os.chdir(cwd)
|
|
self.assertEqual(restored, 0)
|
|
self.assertEqual(unsatisfied, ["bios/absent.bin"])
|
|
|
|
def test_an_unreachable_buildbot_is_not_a_fresh_verdict(self):
|
|
source = (ROOT / "scripts" / "check_buildbot_system.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
self.assertIn(
|
|
'if report.get("error"):',
|
|
source.split("def main(")[-1],
|
|
"main() ignores the error the report carries",
|
|
)
|
|
|
|
def test_a_missing_profile_directory_says_so(self):
|
|
import io as _io
|
|
|
|
from scripts import common as _common
|
|
|
|
stderr = _io.StringIO()
|
|
with contextlib.redirect_stderr(stderr):
|
|
profiles = _common.load_emulator_profiles(
|
|
str(ROOT / "no-such-emulator-dir")
|
|
)
|
|
self.assertEqual(profiles, {})
|
|
self.assertIn("no emulator profile directory", stderr.getvalue())
|
|
|
|
|
|
class TestEntryPointsRunEveryClass(unittest.TestCase):
|
|
"""`unittest.main()` has to sit after the last test class.
|
|
|
|
In this file it sat in the middle, so running it directly discovered only
|
|
the classes defined above it: eight cases never ran that way, while
|
|
`python -m unittest discover` ran all of them. The two roads have to agree.
|
|
"""
|
|
|
|
def test_no_module_calls_main_before_its_last_class(self):
|
|
for path in sorted((ROOT / "tests").glob("test_*.py")):
|
|
lines = path.read_text(encoding="utf-8").splitlines()
|
|
# Column zero only: the same text appears inside this very test as
|
|
# a string literal, and a substring search matched that instead.
|
|
entry = next(
|
|
(n for n, line in enumerate(lines)
|
|
if line.startswith("if __name__ ==")),
|
|
None,
|
|
)
|
|
if entry is None:
|
|
continue
|
|
last_class = max(
|
|
(n for n, line in enumerate(lines) if line.startswith("class ")),
|
|
default=-1,
|
|
)
|
|
self.assertGreater(
|
|
entry,
|
|
last_class,
|
|
f"{path.name} calls unittest.main() before its last class, so "
|
|
"running the file directly skips what follows",
|
|
)
|
|
|
|
|
|
class ScriptsImportThreeWays(unittest.TestCase):
|
|
"""A script is run directly, run as a module, and imported as a package.
|
|
|
|
The three do not agree on what is on the path: only the first form adds
|
|
the scripts directory. Adding the package marker without the bootstrap
|
|
made `import scripts.common` fail on the first sibling import it reached.
|
|
"""
|
|
|
|
def _run(self, *args: str):
|
|
import subprocess
|
|
|
|
return subprocess.run(
|
|
[sys.executable, *args],
|
|
capture_output=True, text=True, cwd=str(ROOT), timeout=300,
|
|
)
|
|
|
|
def test_imported_as_a_package(self):
|
|
result = self._run(
|
|
"-c", "import scripts.common, scripts.verify, scripts.generate_pack"
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr[-400:])
|
|
|
|
def test_run_as_a_module(self):
|
|
result = self._run("-m", "scripts.scraper.libretro_scraper", "--help")
|
|
self.assertEqual(result.returncode, 0, result.stderr[-400:])
|
|
|
|
def test_run_as_a_script(self):
|
|
result = self._run("scripts/list_platforms.py")
|
|
self.assertEqual(result.returncode, 0, result.stderr[-400:])
|
|
|
|
|
|
class ContributorsSurviveAFailedRequest(unittest.TestCase):
|
|
"""The one part of the README that comes from the network.
|
|
|
|
A refused or rate-limited request left the list empty, which deleted the
|
|
section from a published README. It happened during a pipeline run and
|
|
the result was committed, after which the freshness check regenerated the
|
|
section and failed on the difference. Losing the list is a worse answer
|
|
than publishing a stale one.
|
|
"""
|
|
|
|
def _block(self, text: str):
|
|
import tempfile
|
|
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from generate_readme import _existing_contributor_block
|
|
|
|
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as handle:
|
|
handle.write(text)
|
|
name = handle.name
|
|
try:
|
|
return _existing_contributor_block(name)
|
|
finally:
|
|
os.unlink(name)
|
|
|
|
def test_a_published_list_is_read_back(self):
|
|
text = (
|
|
"# Title\n\n## Contributors\n\n"
|
|
'<a href="https://github.com/a"><img src="x" alt="a"></a>\n'
|
|
'<a href="https://github.com/b"><img src="y" alt="b"></a>\n\n'
|
|
"## Community tools\n\nsomething else\n"
|
|
)
|
|
block = self._block(text)
|
|
self.assertEqual(block[0], "## Contributors")
|
|
self.assertIn('alt="a"', "\n".join(block))
|
|
self.assertIn('alt="b"', "\n".join(block))
|
|
self.assertNotIn("Community tools", "\n".join(block))
|
|
self.assertNotIn("something else", "\n".join(block))
|
|
|
|
def test_a_readme_without_the_section_yields_nothing(self):
|
|
self.assertEqual(self._block("# Title\n\n## Other\n\ntext\n"), [])
|
|
|
|
def test_an_unreadable_file_yields_nothing(self):
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from generate_readme import _existing_contributor_block
|
|
|
|
self.assertEqual(_existing_contributor_block("/nonexistent/README.md"), [])
|
|
|
|
def test_the_committed_readme_still_carries_its_contributors(self):
|
|
"""Regenerating offline must never be the reason this disappears."""
|
|
text = (ROOT / "README.md").read_text()
|
|
if "## Contributors" not in text:
|
|
self.skipTest("README carries no contributors section")
|
|
self.assertGreater(
|
|
text.count('<a href="https://github.com/'), 0,
|
|
"the section is present but empty",
|
|
)
|
|
|
|
|
|
class ModuleConstantsDeclaredOnce(unittest.TestCase):
|
|
"""Splitting common.py into modules emitted some constants twice.
|
|
|
|
The values matched, so nothing broke, but the second assignment orphans
|
|
the comment written above the first and leaves two lines to keep in step
|
|
the day a value changes.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _redeclared(path: Path) -> list[str]:
|
|
seen: dict[str, str] = {}
|
|
again = []
|
|
for node in ast.parse(path.read_text()).body:
|
|
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
|
|
continue
|
|
target = node.targets[0]
|
|
if not isinstance(target, ast.Name):
|
|
continue
|
|
value = ast.unparse(node.value)
|
|
if seen.get(target.id) == value:
|
|
again.append(target.id)
|
|
seen[target.id] = value
|
|
return again
|
|
|
|
def test_no_module_assigns_the_same_constant_twice(self):
|
|
offenders = {}
|
|
for path in sorted(ROOT.glob("scripts/**/*.py")) + [ROOT / "install.py"]:
|
|
again = self._redeclared(path)
|
|
if again:
|
|
offenders[path.name] = again
|
|
self.assertEqual(offenders, {})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|