diff --git a/scripts/generate_site.py b/scripts/generate_site.py index dd04f5697..2e8119daf 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -6855,6 +6855,9 @@ def system_pages( _section_label(manufacturer, console), _install_pairs(installs.get(console, []), lead, consumers)) + def region_block(console: str) -> list[str]: + return sitemeasurements.region_section(_REGION_RESULTS.get((slug, console), [])) + def all_sections() -> dict[str, list[Card]]: anchors: set[str] = set() return {console: cards_of(console, anchors) for console in ordered} @@ -6896,6 +6899,7 @@ def system_pages( lines.append("") lines.extend(start_of(console)) lines.extend(install_block(console)) + lines.extend(region_block(console)) lines.extend(_cards_markdown(cards)) lines.append("") lines.extend(_next_block([*_NEXT_FROM_SYSTEMS, json_next])) @@ -6958,6 +6962,7 @@ def system_pages( if console not in moved: summary.extend(start_of(console)) summary.extend(install_block(console)) + summary.extend(region_block(console)) summary.extend(_cards_markdown(cards_of(console, summary_anchors))) summary.append("") continue @@ -7042,6 +7047,8 @@ def system_pages( lines.extend(legend) lines.extend(_romset_note({console: consoles[console]})) lines.extend(install_block(console)) + if index == 0: + lines.extend(region_block(console)) lines.extend(["## Files", ""]) lines.extend(_cards_markdown(part)) lines.append("") @@ -8439,6 +8446,9 @@ _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} # The measurement index items of each emulator, for its Tested block. _MEASUREMENT_ITEMS: dict[str, list[dict]] = {} +# (manufacturer slug, console) -> the region outcomes of its systems +# (region_check.py), shown under the console on its system page. +_REGION_RESULTS: dict[tuple[str, str], list[dict]] = {} # The emulator pages that list each data directory, by ref. _DATA_READERS: dict[str, list[tuple[str, str]]] = {} @@ -11119,6 +11129,13 @@ def _generate(args: argparse.Namespace) -> None: _system_pages_from_data(manufacturers, coverages, db, profiles) ) print(f" {len(_system_page_map)} system IDs mapped to pages") + from siterender import _page_by_norm # noqa: PLC0415 + + _REGION_RESULTS.clear() + for result in sitemeasurements.site_region_results(repo=REPO_URL): + place = _system_page_map.get(result["system"]) or _page_by_norm(result["system"]) + if place: + _REGION_RESULTS.setdefault(place, []).append(result) set_console_sections(manufacturers, program_names(profiles)) from siterender import link_coverage, system_ids # noqa: PLC0415 @@ -11173,7 +11190,9 @@ def _generate(args: argparse.Namespace) -> None: _MEASUREMENT_ITEMS.update(sitemeasurements.by_emulator(measured.index)) print(f" {measured.index['records']} records, {measured.index['count']} tests") write_if_changed(str(docs / sitemeasurements.EXPLORER_PAGE), - sitemeasurements.generate_explorer_page(measured.index) + reference_next()) + sitemeasurements.generate_explorer_page( + measured.index, {sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))}) + + reference_next()) print("Generating static API and bulk data exports...") exports = generate_data_exports( diff --git a/scripts/region_check.py b/scripts/region_check.py new file mode 100644 index 000000000..668f4caea --- /dev/null +++ b/scripts/region_check.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Region checks: which regional BIOS a core reads for a game, and whether +the game boots without it. + +For a system whose profile marks its BIOS entries with `region:`, each test +game of a region (tests/campaign/content.yml) runs through core_probe four +times, each a record in measurements/.jsonl: + + full every regional BIOS the profile declares + own only the entries whose `region:` holds the game's region + others only the entries of the other regions + pack the files of `generate_pack.py --emulator E --region R`, at + the place the pack puts them + +The outcome reads the records: the file the core opened in the full run +(from strace) settles the `region:` claim, the verdicts of `own` and +`others` say whether the BIOS is locked to the region, and `pack` says +whether the pack filtered for the region boots the region's game. + + run the four runs for each region game of one core and system + report the outcome of every region record already in measurements/ + +A maintainer starts a run. Nothing here runs on a timer. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tempfile +import urllib.request +from collections.abc import Callable, Iterable +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import yaml # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent +CONTENT_FILE = ROOT / "tests" / "campaign" / "content.yml" +CORES = ROOT / ".cache" / "region" / "cores" +NIGHTLY = "https://buildbot.libretro.com/nightly/linux/x86_64/latest/{core}_libretro.so.zip" +VARIANTS = ("full", "own", "others", "pack") +SUBJECT = re.compile(r"^region-(?P[a-z0-9_]+)-(?P[a-z0-9-]+)-(?Pfull|own|others|pack)$") +# The territory a test game names, as generate_pack --region takes it. +PACK_REGION = {"japan": "jp", "north-america": "us", "europe": "eu"} +REGION_WORDS = {"japan": "Japanese", "north-america": "US", "europe": "European"} + + +# -------------------------------------------------------------------------- +# Pure rules +# -------------------------------------------------------------------------- + +def regions_of(entry: dict) -> list[str]: + regions = entry.get("region") or [] + return [regions] if isinstance(regions, str) else list(regions) + + +def entry_dest(entry: dict) -> str: + path = entry.get("path") + if isinstance(path, str) and path and not path.endswith("/"): + return path + return entry["name"] + + +def region_entries(profile: dict, system: str) -> list[dict]: + """The BIOS entries of a system that a `region:` ties to territories.""" + out = [] + for entry in profile.get("files") or (): + systems = entry.get("systems") or ([entry["system"]] if entry.get("system") + else profile.get("systems") or []) + if (system in systems and regions_of(entry) and "world" not in regions_of(entry) + and entry.get("category", "bios") == "bios" and entry.get("mode") != "standalone"): + out.append(entry) + return out + + +def plan_variants(entries: Iterable[dict], region: str, + local: Callable[[dict], Path | None]) -> dict[str, dict[str, Path]]: + """The staged files of `full`, `own` and `others`, dest -> file. An entry + the collection does not hold is left out of every variant.""" + held = [(entry, local(entry)) for entry in entries] + held = [(entry, path) for entry, path in held if path] + full = {entry_dest(e): p for e, p in held} + own = {entry_dest(e): p for e, p in held if region in regions_of(e)} + others = {entry_dest(e): p for e, p in held if region not in regions_of(e)} + return {"full": full, "own": own, "others": others} + + +def subject(core: str, game: str, variant: str) -> str: + words = re.findall(r"[a-z0-9]+", game.lower()) + return f"region-{core.lower().replace('-', '_')}-{'-'.join(words)[:60].strip('-')}-{variant}" + + +def opened(record: dict) -> set[str]: + return {item["path"] for variant in record["variants"] for rep in variant["repetitions"] + for item in rep.get("files_opened") or ()} + + +def frames(record: dict) -> list: + rep = record["variants"][0]["repetitions"][0] + return (rep.get("observables") or {}).get("frame_hashes") or [] + + +def verdict(record: dict) -> str: + return record["variants"][0].get("verdict") or "unknown" + + +def outcome(records: dict[str, dict], entries: list[dict], region: str) -> dict: + """What the four records of one game on one core settle. + + `selected` is the regional file the core opened with every one staged. + The `region:` claim is `confirmed` when the regions of what it opened + hold the game's region, `contradicted` when none does, `unread` when it + opened none. Each variant's result is its oracle verdict. While no + reference frame is known for the game, a variant is compared with the + full run instead: `as_full` when it samples the same frames, which is + what the oracle's reference frames will name. + """ + by_dest = {entry_dest(e): e for e in entries} + full = records.get("full") + selected = sorted(opened(full) & set(by_dest)) if full else [] + if not full: + claim = "unrun" + elif not selected: + claim = "unread" + elif any(region in regions_of(by_dest[d]) for d in selected): + claim = "confirmed" + else: + claim = "contradicted" + results: dict[str, dict] = {} + for label in VARIANTS: + record = records.get(label) + if not record: + continue + loaded = all(rep.get("loaded") for v in record["variants"] for rep in v["repetitions"]) + item = {"record": record["id"], "verdict": verdict(record), "loaded": loaded, + "staged": sorted(record["variants"][0]["system_files"])} + if full and label != "full": + item["as_full"] = loaded and frames(record) == frames(full) + results[label] = item + + full_verdict = verdict(full) if full else None + + def boots(label: str) -> bool | None: + item = results.get(label) + if not item: + return None + if item["verdict"] != "unknown": + return item["verdict"] == "boots" + # No verdict: the interim rule, as the full run behaves, and only + # while the full run itself is not shown to fail. + if full_verdict in ("boots", "unknown") and "as_full" in item: + return item["as_full"] + return None + + return { + "region": region, + "selected": [by_dest[d]["name"] for d in selected], + "claim": claim, + "variants": results, + "own_boots": boots("own"), + "others_boot": boots("others"), + "pack_boots": boots("pack"), + "judged_by": "verdict" if full_verdict not in (None, "unknown") else "full_run_frames", + } + + +def sentence(game_region: str, result: dict) -> str: + """The outcome in words, for the site.""" + word = REGION_WORDS.get(game_region, game_region) + own, others = result.get("own_boots"), result.get("others_boot") + if own is None: + return f"The {word} game has no run with its own BIOS yet." + first = f"The {word} BIOS {'boots' if own else 'does not boot'} the {word} game" + if others is None: + return first + "." + if others: + return first + ", and so do the other regions' BIOS." + return first + ", the other regions' BIOS do not." + + +# -------------------------------------------------------------------------- +# Runs +# -------------------------------------------------------------------------- + +def load_content(path: Path = CONTENT_FILE) -> dict[str, list[dict]]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + out: dict[str, list[dict]] = {} + for system, spec in (data.get("systems") or {}).items(): + for item in spec.get("content") or (): + out.setdefault(system, []).append({**item, "romfetch": spec["romfetch"]}) + return out + + +def content_file(item: dict, content_dir: Path) -> Path: + folder = content_dir / item["romfetch"] / item["name"] + if item.get("load") == "cue": + cues = sorted(folder.glob("*.cue")) + if len(cues) != 1: + raise FileNotFoundError(f"{folder}: {len(cues)} .cue files, one expected") + return cues[0] + if item.get("load"): + return folder / item["load"] + files = [p for p in sorted(folder.iterdir()) if p.is_file() and p.name != "identity.json"] + if len(files) != 1: + raise FileNotFoundError(f"{folder}: {len(files)} files, name one with load:") + return files[0] + + +def nightly_core(core: str) -> tuple[Path, str]: + """The buildbot nightly of a core, downloaded once, and its URL.""" + import zipfile # noqa: PLC0415 + + url = NIGHTLY.format(core=core) + target = CORES / f"{core}_libretro.so" + if not target.is_file(): + CORES.mkdir(parents=True, exist_ok=True) + archive = CORES / f"{core}.zip" + with urllib.request.urlopen(url, timeout=300) as response, archive.open("wb") as out: + out.write(response.read()) + with zipfile.ZipFile(archive) as handle: + member = f"{core}_libretro.so" + if member not in handle.namelist(): + raise FileNotFoundError(f"{archive.name}: no {member} inside") + partial = target.with_suffix(".part") + partial.write_bytes(handle.read(member)) + partial.replace(target) + archive.unlink() + return target, url + + +def pack_files(emulator: str, region: str, work: Path) -> dict[str, Path]: + """The files generate_pack puts in the emulator's pack for one region, + extracted, by their path in the pack.""" + from ziptools import safe_extract_zip # noqa: PLC0415 + + out_dir = work / "pack" + subprocess.run([sys.executable, str(ROOT / "scripts" / "generate_pack.py"), "--emulator", emulator, + "--region", PACK_REGION[region], "--output-dir", str(out_dir)], + cwd=ROOT, check=True, capture_output=True, text=True) + zips = sorted(out_dir.glob("*.zip")) + if len(zips) != 1: + raise FileNotFoundError(f"{out_dir}: {len(zips)} packs, one expected") + tree = work / "pack-tree" + safe_extract_zip(str(zips[0]), str(tree)) + return {p.relative_to(tree).as_posix(): p for p in sorted(tree.rglob("*")) if p.is_file() + and not p.name.upper().startswith("README")} + + +def local_resolver() -> Callable[[dict], Path | None]: + from common import load_database, resolve_local_file # noqa: PLC0415 + + db = load_database(str(ROOT / "database.json")) + + def local(entry: dict) -> Path | None: + hint = entry.get("path") if isinstance(entry.get("path"), str) else "" + path, status = resolve_local_file(entry, db, dest_hint=hint or "") + if not path or status in ("not_found", "hash_mismatch"): + return None + found = Path(path) if Path(path).is_absolute() else ROOT / path + return found if found.is_file() else None + + return local + + +def run(args: argparse.Namespace) -> int: + from common import load_emulator_profiles # noqa: PLC0415 + + profile = load_emulator_profiles(str(ROOT / "emulators"))[args.emulator] + entries = region_entries(profile, args.system) + if not entries: + print(f"ERROR: {args.emulator} declares no regional BIOS for {args.system}", file=sys.stderr) + return 2 + games = [g for g in load_content().get(args.system, []) if g.get("region") in PACK_REGION] + if not games: + print(f"ERROR: content.yml has no regional game for {args.system}", file=sys.stderr) + return 2 + core, url = (Path(args.core), args.core_artifact) if args.core else nightly_core(args.core_name) + local = local_resolver() + failures = 0 + for game in games: + load = content_file(game, Path(args.content_dir)) + with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch: + variants = plan_variants(entries, game["region"], local) + variants["pack"] = pack_files(args.emulator, game["region"], Path(scratch)) + for label in args.variants or VARIANTS: + argv = [sys.executable, str(ROOT / "scripts" / "core_probe.py"), "run", + "--core", str(core), "--content", str(load), "--content-label", game["name"], + "--frames", str(game.get("frames", 3600)), + "--frame-hash-every", str(game.get("frame_hash_every", 10)), + "--record", str(ROOT), "--emulator", args.emulator, + "--subject", subject(core.name.removesuffix("_libretro.so"), game["name"], label), + "--require-sandbox", "--timeout", str(args.timeout)] + if url: + argv += ["--core-artifact", url] + for dest, src in sorted(variants[label].items()): + argv += ["--system-file", f"{dest}={src}"] + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, text=True) + status = "recorded" if proc.returncode == 0 else f"exit {proc.returncode}" + print(json.dumps({"game": game["name"], "variant": label, "status": status, + "error": proc.stderr.strip()[-400:] if proc.returncode else None}), + flush=True) + failures += proc.returncode != 0 + return 1 if failures else 0 + + +def collect(root: Path = ROOT, stored_records: list | None = None) -> list[dict]: + """Every region outcome the records under measurements/ hold. A game + listed under several systems (sega-megacd and mega-cd) is judged under + the one whose profile ran it.""" + from common import load_emulator_profiles # noqa: PLC0415 + from measurement_records import live, read_records # noqa: PLC0415 + + content: dict[str, list[tuple[str, dict]]] = {} + for system, items in load_content(root / "tests" / "campaign" / "content.yml").items(): + for item in items: + content.setdefault(item["name"], []).append((system, item)) + profiles = load_emulator_profiles(str(root / "emulators")) + groups: dict[tuple[str, str, str], dict[str, dict]] = {} + for stored in read_records(root) if stored_records is None else stored_records: + match = SUBJECT.match(stored.record.get("subject", "")) + label = ((stored.record.get("inputs") or {}).get("content") or {}).get("label") + if not match or label not in content: + continue + key = (stored.emulator, match["core"], label) + groups.setdefault(key, {})[match["variant"]] = stored.record + out = [] + for (emulator, core, game), records in sorted(groups.items()): + records = {k: v for k, v in records.items() if v["id"] in {r["id"] for r in live(records.values())}} + profile = profiles.get(emulator, {}) + system, item = next(((s, i) for s, i in content[game] if s in (profile.get("systems") or ())), + content[game][0]) + entries = region_entries(profile, system) + result = outcome(records, entries, item["region"]) + out.append({"emulator": emulator, "core": core, "system": system, "game": game, **result, + "sentence": sentence(item["region"], result)}) + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + go = sub.add_parser("run", help="the four runs of each region game, recorded") + go.add_argument("--emulator", required=True, help="the profile, e.g. beetle_psx") + go.add_argument("--system", required=True, help="the system, e.g. sony-playstation") + go.add_argument("--core-name", help="buildbot core name, e.g. mednafen_psx (downloads the nightly)") + go.add_argument("--core", help="a core .so to use instead of the nightly") + go.add_argument("--core-artifact", help="where --core was downloaded from") + go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content")) + go.add_argument("--variants", nargs="*", choices=VARIANTS, help="a subset of the four runs") + go.add_argument("--timeout", type=int, default=900, help="seconds per run") + sub.add_parser("report", help="the outcomes the records hold, as JSON") + args = parser.parse_args(argv) + if args.command == "run": + if not (args.core or args.core_name): + parser.error("run needs --core-name or --core") + return run(args) + print(json.dumps(collect(), indent=2)) + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index ac420682f..81040c407 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -204,7 +204,8 @@ def campaign_section(summary: dict | None) -> list[str]: lines = [ "## Campaign", "", - f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). " + f"A maintainer starts a pass, on an idle computer, and nothing runs by itself " + f"([how]({METHOD_PAGE})). " f"Last pass: {_plain(last['ended_at'][:10])}, {_plain(last['result'])}" + (f" ({_plain(last['detail'])})" if last.get("detail") else "") + ". " f"{int(counts['done'])} tests done, {int(counts['failed'])} failed, {int(counts['pending'])} waiting. " @@ -239,6 +240,125 @@ def campaign_section(summary: dict | None) -> list[str]: return lines +# -------------------------------------------------------------------------- +# Coverage and region results +# -------------------------------------------------------------------------- + +CONTENT_FILE = Path("tests") / "campaign" / "content.yml" + + +def coverage(root: Path, stored: list[mr.Stored]) -> dict: + """Per system of the content list: its games, and whether a record ran + each one. A system no catalog covers comes with the reason and the + error romfetch returned.""" + import yaml # noqa: PLC0415 + + path = root / CONTENT_FILE + if not path.is_file(): + return {"systems": [], "uncovered": []} + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + ran: dict[str, set[str]] = {} + for entry in stored: + label = ((entry.record.get("inputs") or {}).get("content") or {}).get("label") + if label: + ran.setdefault(label, set()).add(entry.emulator) + systems = [] + for system, spec in (data.get("systems") or {}).items(): + games = [{"name": item["name"], "region": item.get("region"), + "emulators": sorted(ran.get(item["name"], ()))} for item in spec.get("content") or ()] + systems.append({"system": system, "games": games, + "tested": any(game["emulators"] for game in games)}) + uncovered = [{"system": system, "want": item.get("want"), "reason": " ".join(str(item["reason"]).split()), + "romfetch": item.get("romfetch") or {}} + for system, item in (data.get("uncovered") or {}).items()] + return {"systems": systems, "uncovered": uncovered} + + +def coverage_section(cov: dict | None, system_names: dict[str, str] | None = None) -> list[str]: + """The Coverage block of the explorer page: tested or not, and why not.""" + if not cov or not (cov.get("systems") or cov.get("uncovered")): + return [] + names = system_names or {} + tested = [s for s in cov["systems"] if s["tested"]] + lines = [ + "## Coverage", + "", + f"Test games come from [`{CONTENT_FILE.as_posix()}`]({REPO}/blob/main/{CONTENT_FILE.as_posix()}), " + f"one per region where a BIOS depends on it. {len(tested)} of {len(cov['systems'])} listed " + f"systems have a recorded run, and {len(cov['uncovered'])} " + f"{'system has' if len(cov['uncovered']) == 1 else 'systems have'} no test game.", + "", + "| System | Games | Tested by |", + "|--------|-------|-----------|", + ] + for item in cov["systems"]: + games = ", ".join(_plain(game["name"]) for game in item["games"]) + by = sorted({emulator for game in item["games"] for emulator in game["emulators"]}) + status = ", ".join(_plain(e) for e in by) if by else "not yet: no run recorded" + lines.append(f"| {_plain(names.get(item['system'], item['system']))} | {games} | {status} |") + lines.append("") + if cov["uncovered"]: + lines += ["Systems without a test game:", "", + "| System | Wanted | Why not | romfetch |", + "|--------|--------|---------|----------|"] + for item in cov["uncovered"]: + code = item["romfetch"].get("code") or "no source" + lines.append(f"| {_plain(names.get(item['system'], item['system']))} | {_plain(item['want'])} | " + f"{_plain(item['reason'])} | exit {_plain(item['romfetch'].get('exit'))}, " + f"{_code(code)} |") + lines.append("") + return lines + + +def region_results(root: Path, stored: list[mr.Stored], source: dict | None) -> list[dict]: + """The region outcomes the records hold (region_check.py), each record + linked at the commit the explorer links.""" + import region_check # noqa: PLC0415 + + lines = {s.record["id"]: s for s in stored} + results = region_check.collect(root, stored) + for result in results: + for item in result["variants"].values(): + found = lines.get(item["record"]) + item["url"] = (permalink(source["repo"], source["commit"], mr.records_path(found.emulator), + found.line) + if found and source and source.get("commit") else None) + return results + + +def site_region_results(root: Path = ROOT, repo: str = REPO) -> list[dict]: + """region_results with the records and commit the explorer uses.""" + stored = mr.read_records(root) + source = {"repo": repo, "commit": records_commit(root) if stored else None} + return region_results(root, stored, source) + + +VARIANT_WORDS = {"full": "every regional BIOS", "own": "its own region's BIOS only", + "others": "the other regions' BIOS only", "pack": "the region's pack"} + + +def region_section(results: list[dict]) -> list[str]: + """What the region runs showed for one system, with the records.""" + if not results: + return [] + lines = ["### Region check", "", + f"Each game ran with {VARIANT_WORDS['own']}, then with {VARIANT_WORDS['others']} " + f"([method](../{METHOD_PAGE})).", ""] + for result in results: + links = ", ".join( + f"[{VARIANT_WORDS[label]}]({item['url']})" if item.get("url") else _plain(VARIANT_WORDS[label]) + for label, item in result["variants"].items()) + read = (f" It reads {', '.join(_code(name) for name in result['selected'])} " + f"when every regional BIOS is there ({_plain(result['claim'])} against the profile)." + if result["selected"] else "") + basis = ("" if result["judged_by"] == "verdict" + else " Judged against the run with every regional BIOS until reference frames are recorded.") + lines.append(f"- **{_plain(result['core'])}**, {_plain(result['game'])}: " + f"{_plain(result['sentence'])}{read}{basis} Records: {links}.") + lines.append("") + return lines + + # -------------------------------------------------------------------------- # SQLite # -------------------------------------------------------------------------- @@ -460,6 +580,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato profiles = load_profiles(emulators_dir) index = build_index(stored, source, profiles, system_names) index["campaign"] = campaign_summary(root, stored, source) + index["coverage"] = coverage(root, stored) paths = [public / INDEX_PATH] (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True) write_if_changed(str(public / INDEX_PATH), @@ -563,7 +684,7 @@ def with_hosts(page: str, index: dict) -> str: return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n")) -def generate_explorer_page(index: dict) -> str: +def generate_explorer_page(index: dict, system_names: dict[str, str] | None = None) -> str: commit = index.get("commit") repo = index.get("repo") or "" source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}" @@ -586,6 +707,7 @@ def generate_explorer_page(index: dict) -> str: "", "", *campaign_section(index.get("campaign")), + *coverage_section(index.get("coverage"), system_names), ]) diff --git a/tests/test_region_check.py b/tests/test_region_check.py new file mode 100644 index 000000000..f83a5b865 --- /dev/null +++ b/tests/test_region_check.py @@ -0,0 +1,192 @@ +"""region_check: the variants a region game runs under and what their +records settle.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import region_check as rc # noqa: E402 + +PROFILE = { + "systems": ["sony-playstation"], + "files": [ + {"name": "scph5500.bin", "region": ["japan"]}, + {"name": "scph5501.bin", "region": ["north-america", "latin-america"]}, + {"name": "scph5502.bin", "region": ["europe", "oceania"]}, + {"name": "openbios.bin"}, + {"name": "ps1_rom.bin", "region": ["world"]}, + {"name": "standalone.bin", "region": ["japan"], "mode": "standalone"}, + {"name": "other.bin", "region": ["japan"], "system": "sony-playstation-2"}, + ], +} + + +def record(rid: str, staged: list[str], opened: list[str], hashes: list, verdict: str = "unknown", + loaded: bool = True) -> dict: + return {"id": rid, "variants": [{ + "label": "default", "verdict": verdict, "system_files": {name: "0" * 40 for name in staged}, + "repetitions": [{"loaded": loaded, "files_opened": [{"path": p} for p in opened], + "observables": {"frame_hashes": hashes}}]}]} + + +class Entries(unittest.TestCase): + def test_only_regional_bios_of_the_system_in_the_libretro_build(self) -> None: + names = [e["name"] for e in rc.region_entries(PROFILE, "sony-playstation")] + self.assertEqual(names, ["scph5500.bin", "scph5501.bin", "scph5502.bin"]) + + def test_variants_split_by_the_game_region(self) -> None: + entries = rc.region_entries(PROFILE, "sony-playstation") + plan = rc.plan_variants(entries, "japan", lambda e: Path("/bios") / e["name"]) + self.assertEqual(sorted(plan["full"]), ["scph5500.bin", "scph5501.bin", "scph5502.bin"]) + self.assertEqual(sorted(plan["own"]), ["scph5500.bin"]) + self.assertEqual(sorted(plan["others"]), ["scph5501.bin", "scph5502.bin"]) + + def test_a_file_the_collection_lacks_is_in_no_variant(self) -> None: + entries = rc.region_entries(PROFILE, "sony-playstation") + plan = rc.plan_variants(entries, "europe", + lambda e: None if e["name"] == "scph5502.bin" else Path(e["name"])) + self.assertEqual(plan["own"], {}) + self.assertNotIn("scph5502.bin", plan["full"]) + + def test_subject_names_core_game_and_variant(self) -> None: + slug = rc.subject("mednafen_psx", "Puzzle Bobble 2 (Japan)", "own") + self.assertEqual(slug, "region-mednafen_psx-puzzle-bobble-2-japan-own") + self.assertRegex(slug, rc.SUBJECT) + + +class Outcome(unittest.TestCase): + ENTRIES = rc.region_entries(PROFILE, "sony-playstation") + ALL = ["scph5500.bin", "scph5501.bin", "scph5502.bin"] + + def test_locked_bios_by_verdict(self) -> None: + records = { + "full": record("f", self.ALL, ["scph5500.bin"], [[0, "a"]], "boots"), + "own": record("o", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"), + "others": record("x", self.ALL[1:], [], [[0, "b"]], "bios_error", loaded=False), + "pack": record("p", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"), + } + result = rc.outcome(records, self.ENTRIES, "japan") + self.assertEqual(result["selected"], ["scph5500.bin"]) + self.assertEqual(result["claim"], "confirmed") + self.assertEqual((result["own_boots"], result["others_boot"], result["pack_boots"]), + (True, False, True)) + self.assertEqual(result["judged_by"], "verdict") + self.assertEqual(rc.sentence("japan", result), + "The Japanese BIOS boots the Japanese game, the other regions' BIOS do not.") + + def test_interim_rule_compares_with_the_full_run(self) -> None: + records = { + "full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"], [10, "b"]]), + "own": record("o", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"], [10, "b"]]), + "others": record("x", ["scph5500.bin", "scph5502.bin"], ["scph5500.bin"], [[0, "a"], [10, "c"]]), + } + result = rc.outcome(records, self.ENTRIES, "north-america") + self.assertEqual(result["judged_by"], "full_run_frames") + self.assertTrue(result["own_boots"]) + self.assertFalse(result["others_boot"]) + self.assertIsNone(result["pack_boots"]) + self.assertEqual(rc.sentence("north-america", result), + "The US BIOS boots the US game, the other regions' BIOS do not.") + + def test_no_interim_answer_when_the_full_run_fails(self) -> None: + records = {"full": record("f", self.ALL, [], [[0, "a"]], "bios_error"), + "own": record("o", ["scph5502.bin"], [], [[0, "a"]])} + result = rc.outcome(records, self.ENTRIES, "europe") + self.assertIsNone(result["own_boots"]) + self.assertEqual(result["claim"], "unread") + + def test_a_core_reading_another_region_contradicts_the_claim(self) -> None: + records = {"full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"]], "boots")} + self.assertEqual(rc.outcome(records, self.ENTRIES, "japan")["claim"], "contradicted") + + def test_region_free_bios(self) -> None: + records = {"full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"]], "boots"), + "own": record("o", ["scph5502.bin"], [], [[0, "a"]], "boots"), + "others": record("x", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"]], "boots")} + result = rc.outcome(records, self.ENTRIES, "europe") + self.assertEqual(rc.sentence("europe", result), + "The European BIOS boots the European game, and so do the other regions' BIOS.") + + def test_no_own_run(self) -> None: + self.assertEqual(rc.sentence("japan", rc.outcome({}, self.ENTRIES, "japan")), + "The Japanese game has no run with its own BIOS yet.") + + +class ContentList(unittest.TestCase): + def test_every_region_system_has_a_game_per_region_or_says_why(self) -> None: + """A system whose BIOS a profile ties to JP, US and EU has a game for + each of the three, or the content list says why it has none.""" + from common import load_emulator_profiles + + import yaml + + data = yaml.safe_load(rc.CONTENT_FILE.read_text(encoding="utf-8")) + content = rc.load_content() + uncovered = data.get("uncovered") or {} + profiles = load_emulator_profiles(str(ROOT / "emulators")) + regional: set[str] = set() + for profile in profiles.values(): + for system in profile.get("systems") or (): + regions = {r for e in rc.region_entries(profile, system) for r in rc.regions_of(e)} + if {"japan", "north-america", "europe"} <= regions: + regional.add(system) + for system in sorted(regional): + if system in uncovered: + continue + held = {item.get("region") for item in content.get(system, [])} + missing = {"japan", "north-america", "europe"} - held + # One cartridge sold in two territories counts for both. + missing -= {r for item in content.get(system, []) for r in item.get("also_regions") or ()} + self.assertFalse(missing, f"{system}: no game for {sorted(missing)}") + + +class SitePages(unittest.TestCase): + def test_region_section_links_every_record(self) -> None: + import sitemeasurements as sm + + records = {"full": record("f", Outcome.ALL, ["scph5500.bin"], [[0, "a"]], "boots"), + "own": record("o", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"), + "others": record("x", Outcome.ALL[1:], [], [[0, "b"]], "bios_error")} + result = rc.outcome(records, Outcome.ENTRIES, "japan") + result.update(core="mednafen_psx", game="Puzzle Bobble 2 (Japan)", + sentence=rc.sentence("japan", result)) + for label, item in result["variants"].items(): + item["url"] = f"https://example.org/{label}" + text = "\n".join(sm.region_section([result])) + self.assertIn("The Japanese BIOS boots the Japanese game, the other regions' BIOS do not.", text) + self.assertIn("[its own region's BIOS only](https://example.org/own)", text) + self.assertIn("scph5500.bin", text) + self.assertNotIn("reference frames are recorded", text) + self.assertEqual(sm.region_section([]), []) + + def test_coverage_names_tested_and_uncovered_systems(self) -> None: + import sitemeasurements as sm + + cov = {"systems": [{"system": "sony-playstation", "tested": True, "games": [ + {"name": "A (USA)", "region": "north-america", "emulators": ["beetle_psx"]}]}, + {"system": "sega-32x", "tested": False, "games": [ + {"name": "B (World)", "region": None, "emulators": []}]}], + "uncovered": [{"system": "msx", "want": "Gradius", "reason": "No MSX catalog.", + "romfetch": {"exit": 1, "code": "unknown_system"}}]} + text = "\n".join(sm.coverage_section(cov, {"sony-playstation": "PlayStation"})) + self.assertIn("1 of 2 listed systems have a recorded run, and 1 system has no test game", text) + self.assertIn("| PlayStation | A (USA) | beetle_psx |", text) + self.assertIn("| sega-32x | B (World) | not yet: no run recorded |", text) + self.assertIn("exit 1, unknown_system", text) + + def test_coverage_reads_the_real_content_list(self) -> None: + import sitemeasurements as sm + + cov = sm.coverage(ROOT, []) + systems = {item["system"] for item in cov["systems"]} + self.assertIn("sony-playstation", systems) + self.assertTrue(all(item["romfetch"].get("exit") for item in cov["uncovered"])) + + +if __name__ == "__main__": + unittest.main()