feat: import game catalogs and header rules

This commit is contained in:
Abdessamad Derraz committed 2026-11-03 16:36:22 +01:00
1 parent 0939cf4d77
commit 8d8689c2e4
47 files changed
+66040

No files matched your search

@@ -0,0 +1,4 @@
<?xml version="1.0"?>
<datafile><header><name>Ignored - System</name><version>1</version></header>
<game name="X (USA)"><rom name="x.bin" size="1" crc="00000001" sha1="6666666666666666666666666666666666666666"/></game>
</datafile>
@@ -0,0 +1,27 @@
<?xml version="1.0"?>
<datafile>
<header>
<name>Nintendo - Super Nintendo Entertainment System</name>
<version>20260707-000000</version>
</header>
<game name="Zeta Quest (USA)">
<description>Zeta Quest (USA)</description>
<rom name="Zeta Quest (USA).sfc" size="1048576" crc="0000BEEF" md5="00000000000000000000000000000000" sha1="1111111111111111111111111111111111111111"/>
</game>
<game name="Alpha Run (Japan)">
<description>Alpha Run (Japan)</description>
<rom name="Alpha Run (Japan).sfc" size="524288" crc="0000CAFE" sha1="2222222222222222222222222222222222222222"/>
</game>
<game name="Both Worlds (USA, Europe) (Rev 1)">
<description>Both Worlds (USA, Europe) (Rev 1)</description>
<rom name="Both Worlds (USA, Europe) (Rev 1).sfc" size="524288" crc="0000CAFE" sha1="3333333333333333333333333333333333333333"/>
</game>
<game name="Everywhere (World)">
<description>Everywhere (World)</description>
<rom name="Everywhere (World).sfc" size="262144" crc="12345678" sha1="4444444444444444444444444444444444444444"/>
</game>
<game name="Untagged Demo">
<description>Untagged Demo</description>
<rom name="Untagged Demo.sfc" size="131072" crc="abcdef01" sha1="5555555555555555555555555555555555555555"/>
</game>
</datafile>
@@ -0,0 +1,24 @@
<?xml version="1.0"?>
<!DOCTYPE datafile PUBLIC "-//Logiqx//DTD ROM Management Datafile//EN" "http://www.logiqx.com/Dats/datafile.dtd">
<datafile>
<header>
<name>Sega - Mega CD &amp; Sega CD</name>
<version>2026-10-23 10-16-53</version>
</header>
<game name="Sonic CD (Europe)">
<serial>4407</serial>
<rom name="Sonic CD (Europe).cue" size="1000" crc="99999999" sha1="7777777777777777777777777777777777777777"/>
<rom name="Sonic CD (Europe) (Track 01).bin" size="129590496" crc="0060ac74" sha1="1ab058c8228af31a50def1b01f1f33961f2f23d7"/>
<rom name="Sonic CD (Europe) (Track 02).bin" size="1000" crc="88888888" sha1="8888888888888888888888888888888888888888"/>
</game>
<game name="Sonic CD (USA)">
<serial>4407, T-0000</serial>
<rom name="Sonic CD (USA).cue" size="1000" crc="99999990" sha1="9999999999999999999999999999999999999999"/>
<rom name="Sonic CD (USA) (Track 01).bin" size="129590496" crc="0060ac75" sha1="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"/>
</game>
<game name="Fresh Cleaner (Japan)">
<serial>M-20001</serial>
<rom name="Fresh Cleaner (Japan).cue" size="100" crc="11111110" sha1="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"/>
<rom name="Fresh Cleaner (Japan) (Track 1).bin" size="6644400" crc="020ca333" sha1="ba09041ae84fc5798a0c88b688ac0c7b9d42b873"/>
</game>
</datafile>
+177
View File
@@ -0,0 +1,177 @@
"""Game catalogs: DAT import, region tags, determinism, floors, schema."""
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
TMP_ROOT = ROOT / "tmp" / "tests"
TMP_ROOT.mkdir(parents=True, exist_ok=True)
FIXTURES = ROOT / "tests" / "fixtures" / "game_dats"
sys.path.insert(0, str(ROOT / "scripts"))
from region import REGIONS # noqa: E402
from scripts.scraper import game_dat_importer as importer # noqa: E402
try:
from jsonschema import Draft202012Validator
except ImportError: # pragma: no cover - CI installs it
Draft202012Validator = None
def run_import(source: str, output: Path, *extra: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, "-m", "scripts.scraper.game_dat_importer", "--source", source,
"--pack", str(FIXTURES), "--output-dir", str(output), *extra],
cwd=ROOT, capture_output=True, text=True, check=False,
)
class RegionTags(unittest.TestCase):
def test_first_territory_group_wins(self) -> None:
self.assertEqual(importer.name_regions("Game (USA) (Rev 1)"), ["north-america"])
self.assertEqual(importer.name_regions("Game (USA, Europe)"), ["europe", "north-america"])
self.assertEqual(importer.name_regions("Game (En,Fr) (Japan)"), ["japan"])
def test_world_absorbs(self) -> None:
self.assertEqual(importer.name_regions("Game (World)"), ["world"])
def test_untagged_and_unmapped(self) -> None:
self.assertIsNone(importer.name_regions("Game (Rev 1)"))
self.assertIsNone(importer.name_regions("Game (South Africa)"))
def test_extra_words_fall_under_their_super_region(self) -> None:
self.assertEqual(importer.name_regions("Game (Scandinavia)"), ["europe"])
def test_every_mapping_is_vocabulary(self) -> None:
targets = set(importer._TERRITORY.values())
self.assertLessEqual(targets, set(REGIONS))
class FixtureImport(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
self.out = Path(self.temp.name)
def tearDown(self) -> None:
self.temp.cleanup()
def test_nointro_rows(self) -> None:
result = run_import("no-intro", self.out)
self.assertEqual(result.returncode, 0, result.stderr)
files = sorted(p.relative_to(self.out).as_posix() for p in self.out.rglob("*.json"))
self.assertEqual(files, ["no-intro/nintendo-snes.json"])
doc = json.loads((self.out / files[0]).read_text())
self.assertEqual(doc["fields"], ["crc32", "size", "sha1", "region", "title"])
self.assertEqual(len(doc["games"]), 5)
rows = {row[4]: row for row in doc["games"]}
self.assertEqual(rows["Zeta Quest (USA)"][:3],
["0000beef", 1048576, "1" * 40])
self.assertEqual(doc["regions"][rows["Both Worlds (USA, Europe) (Rev 1)"][3]],
["europe", "north-america"])
self.assertIsNone(rows["Untagged Demo"][3])
self.assertEqual([r[0] for r in doc["games"]], sorted(r[0] for r in doc["games"]))
self.assertIn("freely used", doc["licence"]["quote"])
self.assertEqual(doc["licence"]["mirror"]["licence"], "MIT")
def test_redump_first_data_track_and_serials(self) -> None:
result = run_import("redump", self.out)
self.assertEqual(result.returncode, 0, result.stderr)
doc = json.loads((self.out / "redump" / "sega-megacd.json").read_text())
rows = {row[4]: row for row in doc["games"]}
self.assertEqual(rows["Sonic CD (Europe)"][:2], ["0060ac74", 129590496])
self.assertEqual(rows["Sonic CD (USA)"][5], ["4407", "T-0000"])
self.assertEqual(doc["archive"]["dat_date"], "2026-10-23")
self.assertIn("public domain", doc["licence"]["quote"])
self.assertIn("serial_note", doc)
coverage = importer.coverage(doc)
self.assertEqual(coverage["serials"], 3)
self.assertEqual(coverage["multi_region_serials"], 1)
def test_two_imports_are_byte_identical(self) -> None:
first, second = self.out / "a", self.out / "b"
for out in (first, second):
for source in ("no-intro", "redump"):
self.assertEqual(run_import(source, out).returncode, 0)
for path in first.rglob("*.json"):
twin = second / path.relative_to(first)
self.assertEqual(path.read_bytes(), twin.read_bytes(), path.name)
def test_floor_refuses_a_short_dat(self) -> None:
result = run_import("no-intro", self.out, "--floor", "--dry-run")
self.assertEqual(result.returncode, 1)
self.assertIn("floor 3920", result.stderr)
self.assertEqual(list(self.out.rglob("*.json")), [])
def test_offline_without_cache_names_the_remedy(self) -> None:
result = subprocess.run(
[sys.executable, "-m", "scripts.scraper.game_dat_importer", "--source", "redump",
"--cache-dir", str(self.out / "empty"), "--output-dir", str(self.out)],
cwd=ROOT, capture_output=True, text=True, check=False,
)
self.assertEqual(result.returncode, 1)
self.assertIn("--fetch", result.stderr)
class Selection(unittest.TestCase):
def test_every_system_is_a_repository_system(self) -> None:
from romfolders import SystemResolver
ids = SystemResolver.from_repo().ids
wanted = [s for s, _ in importer.NOINTRO_DATS.values()]
wanted += [s for _, s, _ in importer.REDUMP_DATS.values()]
self.assertEqual(len(wanted), len(set(wanted)))
self.assertEqual([s for s in wanted if s not in ids], [])
self.assertEqual(len(importer.NOINTRO_DATS), 29)
self.assertEqual(len(importer.REDUMP_DATS), 9)
@unittest.skipIf(Draft202012Validator is None, "jsonschema not installed")
class CommittedCatalogs(unittest.TestCase):
def setUp(self) -> None:
schema = json.loads((ROOT / "schemas" / "game-catalog.schema.json").read_text())
self.validator = Draft202012Validator(schema)
self.paths = sorted((ROOT / "games" / "catalogs").glob("*/*.json"))
def test_every_catalog_matches_the_schema(self) -> None:
self.assertEqual(len(self.paths), 38)
for path in self.paths:
doc = json.loads(path.read_text())
errors = [e.message for e in self.validator.iter_errors(doc)]
self.assertEqual(errors, [], path.name)
def test_committed_catalogs_meet_their_floor(self) -> None:
for path in self.paths:
doc = json.loads(path.read_text())
self.assertEqual(path.stem, doc["system"])
self.assertGreaterEqual(importer.coverage(doc)["games"], importer.floor_of(doc),
path.name)
def test_region_slugs_are_vocabulary(self) -> None:
for path in self.paths:
doc = json.loads(path.read_text())
for regions in doc["regions"]:
self.assertLessEqual(set(regions), set(REGIONS), path.name)
def test_rewrite_is_a_no_op(self) -> None:
for path in self.paths[:3]:
doc = json.loads(path.read_text())
self.assertEqual(importer.dump_catalog(doc), path.read_text())
def test_fixture_output_matches_the_schema(self) -> None:
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
out = Path(directory)
for source in ("no-intro", "redump"):
self.assertEqual(run_import(source, out).returncode, 0)
for path in out.rglob("*.json"):
errors = [e.message for e in self.validator.iter_errors(json.loads(path.read_text()))]
self.assertEqual(errors, [], path.name)
if __name__ == "__main__":
unittest.main()
+211
View File
@@ -0,0 +1,211 @@
"""Header readers: schema, references between readers and cores, pinned citations."""
from __future__ import annotations
import json
import re
import sys
import unittest
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from common import load_emulator_profiles # noqa: E402
from region import REGIONS # noqa: E402
try:
from jsonschema import Draft202012Validator
except ImportError: # pragma: no cover - CI installs it
Draft202012Validator = None
HEADERS = ROOT / "games" / "headers.yml"
_CITATION = re.compile(r"^([\w.-]+/[\w.-]+)@([0-9a-f]{8,40}) ")
# Tools the containers cite that are not emulators: no profile pins them.
TOOL_REPOS = {"python/cpython", "ip7z/7zip"}
def load() -> dict:
return yaml.safe_load(HEADERS.read_text(encoding="utf-8"))
def pins(profile: dict) -> set[str]:
out: set[str] = set()
for key in ("source_commit", "upstream_commit"):
value = profile.get(key)
if isinstance(value, dict):
out.update(str(v) for v in value.values() if v)
elif value:
out.add(str(value))
return out
def outcomes(node: object) -> list[list[str]]:
"""Every region list anywhere below a node."""
found: list[list[str]] = []
if isinstance(node, dict):
for key, value in node.items():
if key == "regions" and isinstance(value, list):
found.append(value)
else:
found += outcomes(value)
elif isinstance(node, list):
if node and all(isinstance(v, str) and v in REGIONS for v in node):
found.append(node)
for value in node:
found += outcomes(value)
return found
def decode(reader: dict, raw: object) -> object:
"""A reference decoder for the kinds whose rule is a plain lookup."""
rule = reader["decode"]
kind = rule["kind"]
for entry in rule["values"]:
value = entry["value"]
hit = (
(kind == "exact-prefix" and str(raw).startswith(value))
or (kind == "prefix-nocase" and str(raw).lower().startswith(value))
or (kind == "integer" and raw == value)
or (kind == "char-at" and len(str(raw)) > rule["index"]
and str(raw)[rule["index"]] == value)
)
if hit:
return entry.get("regions") or entry.get("standard") or entry.get("label")
return rule["otherwise"]
def check(cond: dict, data: bytes) -> bool:
"""A reference evaluator for the byte, text and size tests of a predicate."""
if "all" in cond:
return all(check(c, data) for c in cond["all"])
if "any" in cond:
return any(check(c, data) for c in cond["any"])
if "file_size" in cond:
bounds = cond["file_size"]
return bounds.get("min", 0) <= len(data) <= bounds.get("max", len(data))
offset = cond["offset"]
if "text" in cond or "not_text" in cond:
want = cond.get("text", cond.get("not_text"))
hit = data[offset:offset + len(want)] == want.encode("latin-1")
return hit if "text" in cond else not hit
value = data[offset] & cond.get("mask", 0xFF)
if "equals" in cond:
return value == cond["equals"]
if "not_equals" in cond:
return value != cond["not_equals"]
return value in cond["in"]
class HeaderReaders(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.doc = load()
cls.profiles = load_emulator_profiles(str(ROOT / "emulators"))
@unittest.skipIf(Draft202012Validator is None, "jsonschema not installed")
def test_schema(self) -> None:
schema = json.loads((ROOT / "schemas" / "game-headers.schema.json").read_text())
errors = [f"{list(e.path)}: {e.message}" for e in Draft202012Validator(schema).iter_errors(self.doc)]
self.assertEqual(errors, [])
def test_core_order_names_readers_of_its_system(self) -> None:
for system, block in self.doc["systems"].items():
for core, spec in block["cores"].items():
missing = [r for r in spec["order"] if r not in block["readers"]]
self.assertEqual(missing, [], f"{system}/{core}")
def test_every_core_is_a_profile(self) -> None:
for system, block in self.doc["systems"].items():
for core in block["cores"]:
self.assertIn(core, self.profiles, system)
def test_every_system_is_a_repository_system(self) -> None:
from romfolders import SystemResolver
ids = SystemResolver.from_repo().ids
self.assertEqual([s for s in self.doc["systems"] if s not in ids], [])
def test_regions_are_vocabulary(self) -> None:
for regions in outcomes(self.doc["systems"]):
self.assertLessEqual(set(regions), set(REGIONS), regions)
def test_core_citations_name_the_profile_pin(self) -> None:
for system, block in self.doc["systems"].items():
for core, spec in block["cores"].items():
match = _CITATION.match(spec["source_ref"])
self.assertIsNotNone(match, f"{system}/{core}")
sha = match.group(2)
self.assertTrue(any(p.startswith(sha) for p in pins(self.profiles[core])),
f"{system}/{core}: {sha} is not a pin of the profile")
def test_reader_citations_name_a_profile_pin(self) -> None:
every_pin = {p for profile in self.profiles.values() for p in pins(profile)}
refs = [(f"{s}/{r}", spec["source_ref"]) for s, block in self.doc["systems"].items()
for r, spec in block["readers"].items()]
refs += [(f"containers/{c}", spec["source_ref"]) for c, spec in self.doc["containers"].items()]
for label, ref in refs:
match = _CITATION.match(ref)
self.assertIsNotNone(match, label)
if match.group(1) in TOOL_REPOS:
continue
self.assertTrue(any(p.startswith(match.group(2)) for p in every_pin),
f"{label}: {match.group(2)} pins no profile")
def test_reference_decodes(self) -> None:
psx = self.doc["systems"]["sony-playstation"]["readers"]
self.assertEqual(decode(psx["psx-serial-prefix"], "SLUS_012.34"), ["north-america"])
self.assertEqual(decode(psx["psx-serial-prefix"], "SCES_000.01"), ["europe"])
self.assertEqual(decode(psx["psx-serial-prefix"], "XXXX_000.01"), "unknown")
self.assertEqual(decode(psx["psx-serial-letter"], "SLPS_012.04"), ["japan"])
self.assertEqual(decode(psx["psx-licence-offset60"], "Euro pe "), ["europe"])
mcd = self.doc["systems"]["sega-megacd"]["readers"]["mcd-security-code"]
self.assertEqual(decode(mcd, 0x64), ["europe"])
self.assertEqual(decode(mcd, 0x7A), ["north-america"])
gc = self.doc["systems"]["nintendo-gamecube"]["readers"]["gc-region"]
self.assertEqual(decode(gc, 3), "unknown")
n64 = self.doc["systems"]["nintendo-64"]["readers"]["n64-country"]
self.assertEqual(decode(n64, 0x45), {"standard": "ntsc"})
self.assertEqual(decode(n64, 0x50), "pal")
def test_reader_ids_are_unique(self) -> None:
ids = [r for block in self.doc["systems"].values() for r in block["readers"]]
self.assertEqual(len(ids), len(set(ids)))
def test_every_reader_extras_names_exists(self) -> None:
extras = yaml.safe_load((ROOT / "games" / "extras.yml").read_text(encoding="utf-8"))
defined = {r for block in self.doc["systems"].values() for r in block["readers"]}
named = {
entry["trigger"]["header"]["reader"]
for entry in extras["extras"]
if "header" in entry.get("trigger", {})
}
self.assertTrue(named)
self.assertEqual(sorted(named - defined), [])
def test_predicate_readers_decode_reference_bytes(self) -> None:
systems = self.doc["systems"]
sgb = systems["nintendo-sgb"]["readers"]["gb-sgb-flag"]["decode"]
header = bytearray(0x150)
header[0x146], header[0x14B] = 0x03, 0x33
self.assertTrue(all(check(c, bytes(header)) for c in sgb["all"]))
header[0x143] = 0x80
self.assertFalse(all(check(c, bytes(header)) for c in sgb["all"]))
sufami = systems["nintendo-sufami-turbo"]["readers"]["sufami-magic"]["decode"]
cart = bytearray(0x80000)
cart[0:14] = b"BANDAI SFC-ADX"
self.assertTrue(all(check(c, bytes(cart)) for c in sufami["all"]))
cart[0x10:0x1E] = b"SFC-ADX BACKUP"
self.assertFalse(all(check(c, bytes(cart)) for c in sufami["all"]))
def test_cores_that_pick_no_bios_read_nothing_for_it(self) -> None:
for system, block in self.doc["systems"].items():
for core, spec in block["cores"].items():
if spec.get("selects_bios") is False:
self.assertNotIn("preference", spec, f"{system}/{core}")
if __name__ == "__main__":
unittest.main()