mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 12:19:13 -05:00
feat: keep romsets holding a missing catalog dump
The recipe snapshot kept only the archives a profile or platform names, so provenance_report --missing named no set for dumps a public romset carries (fm7740sx holds the FM77AV40SX DICROM, EXTSUB and KANJI2 ROMs). The importer now also keeps the sets holding a catalog entry the collection lacks. Reimported from mame0289 and the cached FBNeo DATs: MAME 1726 to 2148 recipes, FBNeo 104 to 111, none dropped.
This commit is contained in:
Abdessamad Derraz committed
1 parent
905316061e
commit
8f86842117
5 files changed
+24339
-19
No files matched your search
+228
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"source": "fbneo",
|
||||
"imported_at": "2026-10-04",
|
||||
"imported_at": "2026-10-21",
|
||||
"dats": {
|
||||
"FinalBurn Neo - Arcade Games": "1.0.0.03",
|
||||
"FinalBurn Neo - Astrocade Home Computer Games": "1.0.0.03",
|
||||
@@ -1326,6 +1326,41 @@
|
||||
"FinalBurn Neo - Arcade Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Arcade Games",
|
||||
"name": "invaders.zip",
|
||||
"set": "invaders",
|
||||
"description": "Space Invaders / Space Invaders M",
|
||||
"members": [
|
||||
{
|
||||
"name": "9316b-0854_m739e.e1",
|
||||
"size": 2048,
|
||||
"crc32": "14e538b0",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "9316b-0855_m739f.f1",
|
||||
"size": 2048,
|
||||
"crc32": "0ccead96",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "9316b-0856_m739g.g1",
|
||||
"size": 2048,
|
||||
"crc32": "6bfaca4a",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "9316b-0869_m739h.h1",
|
||||
"size": 2048,
|
||||
"crc32": "734f5ad8",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - Arcade Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Arcade Games",
|
||||
"name": "irrmaze.zip",
|
||||
@@ -3536,6 +3571,65 @@
|
||||
"FinalBurn Neo - Arcade Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Arcade Games",
|
||||
"name": "omegrace.zip",
|
||||
"set": "omegrace",
|
||||
"description": "Omega Race (set 1)",
|
||||
"members": [
|
||||
{
|
||||
"name": "dvgprom.bin",
|
||||
"size": 256,
|
||||
"crc32": "d481e958",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.e1",
|
||||
"size": 2048,
|
||||
"crc32": "1d0fdf3a",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.f1",
|
||||
"size": 2048,
|
||||
"crc32": "d44c0814",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.j7",
|
||||
"size": 4096,
|
||||
"crc32": "8e8d4b54",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.k7",
|
||||
"size": 4096,
|
||||
"crc32": "6d10f197",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.l7",
|
||||
"size": 4096,
|
||||
"crc32": "edcd7a7d",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "omega.m7",
|
||||
"size": 4096,
|
||||
"crc32": "0424d46e",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "sound.k5",
|
||||
"size": 2048,
|
||||
"crc32": "7d426017",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - Arcade Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Arcade Games",
|
||||
"name": "pgm.zip",
|
||||
@@ -7703,6 +7797,47 @@
|
||||
"FinalBurn Neo - ColecoVision Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - ColecoVision Games",
|
||||
"name": "omegrace.zip",
|
||||
"set": "omegrace",
|
||||
"description": "Omega Race (USA, Euro)",
|
||||
"members": [
|
||||
{
|
||||
"name": "Omega Race (USA, Euro)(1983)(Coleco).rom",
|
||||
"size": 16384,
|
||||
"crc32": "0a0511c7",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "coleco.rom",
|
||||
"size": 8192,
|
||||
"crc32": "3aa93ef3",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "colecoa.rom",
|
||||
"size": 8192,
|
||||
"crc32": "39bb16fc",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "czz50.rom",
|
||||
"size": 16384,
|
||||
"crc32": "4999abc6",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "svi603.rom",
|
||||
"size": 8192,
|
||||
"crc32": "19e91b82",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - ColecoVision Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - ColecoVision Games",
|
||||
"name": "pv2000.zip",
|
||||
@@ -8116,6 +8251,41 @@
|
||||
"FinalBurn Neo - MSX 1 Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - MSX 1 Games",
|
||||
"name": "skydiver.zip",
|
||||
"set": "skydiver",
|
||||
"description": "Sky Diver (Japan)",
|
||||
"members": [
|
||||
{
|
||||
"name": "Sky Diver (Japan, EN)(1984)(Hudson Soft)[BLOAD'CAS-',R].cas",
|
||||
"size": 11734,
|
||||
"crc32": "93a40a2c",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "kanji.rom",
|
||||
"size": 262144,
|
||||
"crc32": "1f6406fb",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "msx.rom",
|
||||
"size": 32768,
|
||||
"crc32": "a317e6b4",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "msxj.rom",
|
||||
"size": 32768,
|
||||
"crc32": "071135e0",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - MSX 1 Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Megadrive Games",
|
||||
"name": "bubsys.zip",
|
||||
@@ -8150,6 +8320,40 @@
|
||||
"FinalBurn Neo - Megadrive Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Megadrive Games",
|
||||
"name": "ggenie1.zip",
|
||||
"set": "ggenie1",
|
||||
"description": "Game Genie (Euro, USA)",
|
||||
"members": [
|
||||
{
|
||||
"name": "sgr001+sgr002.bin",
|
||||
"size": 32768,
|
||||
"crc32": "5f293e4c",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - Megadrive Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - Megadrive Games",
|
||||
"name": "warrior.zip",
|
||||
"set": "warrior",
|
||||
"description": "Warrior of Rome (USA)",
|
||||
"members": [
|
||||
{
|
||||
"name": "warrior of rome (usa).bin",
|
||||
"size": 1048576,
|
||||
"crc32": "5be10c6a",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - Megadrive Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - NES Games",
|
||||
"name": "galaga.zip",
|
||||
@@ -8555,6 +8759,29 @@
|
||||
"FinalBurn Neo - ZX Spectrum Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - ZX Spectrum Games",
|
||||
"name": "invaders.zip",
|
||||
"set": "invaders",
|
||||
"description": "Invaders (16K)",
|
||||
"members": [
|
||||
{
|
||||
"name": "Invaders 16K (1983)(DK'Tronics).tap",
|
||||
"size": 6795,
|
||||
"crc32": "a6263eb9",
|
||||
"sha1": ""
|
||||
},
|
||||
{
|
||||
"name": "spectrum.rom",
|
||||
"size": 16384,
|
||||
"crc32": "ddee531f",
|
||||
"sha1": ""
|
||||
}
|
||||
],
|
||||
"dats": [
|
||||
"FinalBurn Neo - ZX Spectrum Games"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dat": "FinalBurn Neo - ZX Spectrum Games",
|
||||
"name": "mach3.zip",
|
||||
|
||||
+24005
-1
File diff suppressed because it is too large.
Load diff
@@ -15,7 +15,9 @@ in its repository, so both are fetched without a browser:
|
||||
|
||||
Only sets the collection has a reason to know are kept, so the committed
|
||||
snapshot stays small: the archive names that emulator profiles or platform
|
||||
BIOS lists reference. ``--all`` keeps everything.
|
||||
BIOS lists reference, and the sets holding a dump-catalog entry the
|
||||
collection lacks (provenance_report names the set as where to find it).
|
||||
``--all`` keeps everything.
|
||||
|
||||
Members without a CRC32 are undumped. A real romset does not carry them, so
|
||||
they are dropped from the recipe rather than disqualifying the whole set.
|
||||
@@ -42,7 +44,7 @@ from ..common import (
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
)
|
||||
from ..dumpcatalog import build_snapshot, write_snapshot
|
||||
from ..dumpcatalog import build_snapshot, load_provenance_snapshots, write_snapshot
|
||||
from .logiqx_parser import LogiqxDat, parse_logiqx
|
||||
|
||||
MAX_MEMBER_SIZE = 200 * 1024 * 1024
|
||||
@@ -74,8 +76,45 @@ def relevant_archives(emulators_dir: str, platforms_dir: str) -> set[str]:
|
||||
return names
|
||||
|
||||
|
||||
Targets = dict[tuple[str, int], set[str]]
|
||||
|
||||
|
||||
def catalog_targets(provenance_dir: str, db_path: str) -> Targets:
|
||||
"""(crc32, size) -> sha1 of each dump-catalog entry the collection lacks.
|
||||
|
||||
fm7740sx carries DICROM.rom, EXTSUB.rom and KANJI2.rom, which No-Intro
|
||||
lists and the collection misses: filtered on archive names alone, the
|
||||
snapshot never held the set and the missing report named no source.
|
||||
"""
|
||||
with open(db_path, encoding="utf-8") as handle:
|
||||
held = set(json.load(handle).get("files", {}))
|
||||
targets: Targets = {}
|
||||
for snapshot in load_provenance_snapshots(provenance_dir).values():
|
||||
for entry in snapshot.get("entries", []):
|
||||
crc, size = str(entry.get("crc32") or "").lower(), entry.get("size")
|
||||
sha1 = str(entry.get("sha1") or "").lower()
|
||||
if not crc or size is None or (sha1 and sha1 in held):
|
||||
continue
|
||||
targets.setdefault((crc.zfill(8), int(size)), set()).add(sha1)
|
||||
return targets
|
||||
|
||||
|
||||
def holds_target(members: list[dict], targets: Targets | None) -> bool:
|
||||
"""Whether a set carries a missing catalog dump. A sha1 on both sides
|
||||
must agree: crc32 and size alone collide across 40 000 ROMs."""
|
||||
if not targets:
|
||||
return False
|
||||
for member in members:
|
||||
key = (str(member.get("crc32") or "").lower().zfill(8), int(member.get("size") or 0))
|
||||
wanted = targets.get(key)
|
||||
sha1 = str(member.get("sha1") or "").lower()
|
||||
if wanted is not None and (not sha1 or "" in wanted or sha1 in wanted):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def recipe_entries(
|
||||
dat: LogiqxDat, keep: set[str] | None = None
|
||||
dat: LogiqxDat, keep: set[str] | None = None, targets: Targets | None = None
|
||||
) -> list[dict]:
|
||||
"""Group a parsed DAT into one recipe per set."""
|
||||
by_set: dict[str, list] = {}
|
||||
@@ -87,7 +126,9 @@ def recipe_entries(
|
||||
entries: list[dict] = []
|
||||
for set_name, roms in sorted(by_set.items()):
|
||||
archive = f"{set_name}.zip"
|
||||
if keep is not None and archive not in keep:
|
||||
if keep is not None and archive not in keep and not holds_target(
|
||||
[{"crc32": r.crc32, "size": r.size, "sha1": r.sha1} for r in roms], targets
|
||||
):
|
||||
continue
|
||||
entries.append(
|
||||
{
|
||||
@@ -149,9 +190,12 @@ def _machine_roms(blob: bytes) -> list[dict]:
|
||||
|
||||
|
||||
def _scan_listxml(
|
||||
stream: io.BufferedIOBase, selector: Callable[[str], bool]
|
||||
stream: io.BufferedIOBase,
|
||||
selector: Callable[[str], bool],
|
||||
targets: Targets | None = None,
|
||||
) -> dict[str, tuple[str | None, list[dict]]]:
|
||||
"""Stream a MAME -listxml document, keeping the machines *selector* wants.
|
||||
"""Stream a MAME -listxml document, keeping the machines *selector* wants
|
||||
and those holding one of *targets*.
|
||||
|
||||
The document is 311 MB for MAME 0.289, so it is never held whole: the
|
||||
reader keeps a sliding window just large enough to close the element it
|
||||
@@ -170,19 +214,22 @@ def _scan_listxml(
|
||||
if end == -1:
|
||||
break
|
||||
name = match.group(1).decode()
|
||||
if selector(name):
|
||||
parent = _ROMOF.search(match.group(2).decode())
|
||||
found[name] = (
|
||||
parent.group(1) if parent else None,
|
||||
_machine_roms(buffer[match.end() : end]),
|
||||
)
|
||||
wanted = selector(name)
|
||||
if wanted or targets:
|
||||
roms = _machine_roms(buffer[match.end() : end])
|
||||
if wanted or holds_target(roms, targets):
|
||||
parent = _ROMOF.search(match.group(2).decode())
|
||||
found[name] = (parent.group(1) if parent else None, roms)
|
||||
position = end + len(b"</machine>")
|
||||
buffer = buffer[position:] if position else buffer[-(2 * LISTXML_CHUNK) :]
|
||||
return found
|
||||
|
||||
|
||||
def listxml_entries(
|
||||
open_stream: Callable[[], io.BufferedIOBase], label: str, keep: set[str] | None
|
||||
open_stream: Callable[[], io.BufferedIOBase],
|
||||
label: str,
|
||||
keep: set[str] | None,
|
||||
targets: Targets | None = None,
|
||||
) -> list[dict]:
|
||||
"""Recipes from a MAME -listxml document, parents resolved.
|
||||
|
||||
@@ -193,7 +240,7 @@ def listxml_entries(
|
||||
"""
|
||||
with open_stream() as stream:
|
||||
wanted = _scan_listxml(
|
||||
stream, lambda name: keep is None or f"{name}.zip" in keep
|
||||
stream, lambda name: keep is None or f"{name}.zip" in keep, targets
|
||||
)
|
||||
requested = set(wanted)
|
||||
parents = {
|
||||
@@ -484,6 +531,15 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"--all", action="store_true", help="keep every set, not only referenced ones"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--provenance-dir", default="provenance",
|
||||
help="dump catalogs whose missing entries keep the sets holding them "
|
||||
"(default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db", default="database.json",
|
||||
help="collection database the catalogs are compared with (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="print what would change and write nothing",
|
||||
@@ -511,6 +567,7 @@ def main() -> int:
|
||||
if args.all
|
||||
else relevant_archives(args.emulators_dir, args.platforms_dir)
|
||||
)
|
||||
targets = None if args.all else catalog_targets(args.provenance_dir, args.db)
|
||||
|
||||
dats: dict[str, str] = {}
|
||||
entries: list[dict] = []
|
||||
@@ -518,7 +575,7 @@ def main() -> int:
|
||||
|
||||
if _is_listxml(pack):
|
||||
label = f"MAME {pack.stem.replace('lx', '')}"
|
||||
entries = listxml_entries(_listxml_opener(pack), label, keep)
|
||||
entries = listxml_entries(_listxml_opener(pack), label, keep, targets)
|
||||
dats[label] = pack.stem
|
||||
print(f" {label}: {len(entries)} sets retenus")
|
||||
members = sum(len(entry["members"]) for entry in entries)
|
||||
@@ -539,7 +596,7 @@ def main() -> int:
|
||||
if not dat.roms:
|
||||
skipped += 1
|
||||
continue
|
||||
found = recipe_entries(dat, keep)
|
||||
found = recipe_entries(dat, keep, targets)
|
||||
if not found:
|
||||
continue
|
||||
dats[dat.name or member] = dat.version
|
||||
|
||||
@@ -28,7 +28,9 @@ from romset_recipes import ( # noqa: E402
|
||||
write_reconstructions,
|
||||
)
|
||||
from scripts.scraper.romset_dat_importer import ( # noqa: E402
|
||||
catalog_targets,
|
||||
compact_entries,
|
||||
holds_target,
|
||||
listxml_entries,
|
||||
git_blob_sha,
|
||||
merge_snapshot,
|
||||
@@ -386,6 +388,32 @@ class ListXmlRecipes(unittest.TestCase):
|
||||
[m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
|
||||
)
|
||||
|
||||
def test_a_set_holding_a_missing_catalog_dump_is_kept(self):
|
||||
"""fm7740sx carries three No-Intro dumps the collection lacks. No
|
||||
profile names it, so the missing report named no place to find them."""
|
||||
targets = {(_crc(ROM_B), len(ROM_B)): {""}}
|
||||
entries = listxml_entries(
|
||||
self._opener(self._document()), "MAME 0.289", {"other.zip"}, targets
|
||||
)
|
||||
self.assertEqual([entry["name"] for entry in entries], ["childset.zip"])
|
||||
|
||||
def test_a_catalog_sha1_must_agree(self):
|
||||
member = {"crc32": _crc(ROM_B), "size": len(ROM_B), "sha1": "a" * 40}
|
||||
self.assertFalse(holds_target([member], {(_crc(ROM_B), len(ROM_B)): {"b" * 40}}))
|
||||
self.assertTrue(holds_target([member], {(_crc(ROM_B), len(ROM_B)): {"a" * 40}}))
|
||||
|
||||
def test_catalog_targets_leave_out_what_the_collection_holds(self):
|
||||
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
|
||||
root = Path(directory)
|
||||
(root / "prov").mkdir()
|
||||
(root / "prov" / "tosec.json").write_text(json.dumps({"source": "tosec", "entries": [
|
||||
{"name": "held.rom", "crc32": "1", "size": 4, "sha1": "a" * 40},
|
||||
{"name": "lacked.rom", "crc32": "2", "size": 4, "sha1": "b" * 40},
|
||||
]}))
|
||||
(root / "db.json").write_text(json.dumps({"files": {"a" * 40: {}}}))
|
||||
targets = catalog_targets(str(root / "prov"), str(root / "db.json"))
|
||||
self.assertEqual(targets, {("00000002", 4): {"b" * 40}})
|
||||
|
||||
def test_upstream_blobs_are_recorded_and_replaced(self):
|
||||
"""The snapshot names the DAT blobs it was read from, so a later
|
||||
check can tell whether upstream moved without importing again."""
|
||||
|
||||
+5
-1
@@ -738,7 +738,11 @@ python -m scripts.scraper.romset_dat_importer --source fbneo --fetch
|
||||
Snapshots land in `recipes/`, not `provenance/`: the latter holds dump catalogues
|
||||
and a recipe is not one. Versions accumulate rather than replace each other,
|
||||
because a platform pins the archive of whichever version its list was built
|
||||
against. Only sets a profile or platform references are kept, a `romof` parent's
|
||||
against. Only sets a profile or platform references are kept, plus the sets
|
||||
holding a dump-catalog entry the collection lacks (crc32 and size, and sha1
|
||||
when both sides carry one), so `provenance_report.py --missing` can name the
|
||||
set that ships it: MAME's `fm7740sx` carries the FM77AV40SX `DICROM.rom`,
|
||||
`EXTSUB.rom` and `KANJI2.rom` that No-Intro lists. A `romof` parent's
|
||||
members are merged into the child, and members with no CRC32 are dropped: those
|
||||
are undumped and a real romset does not carry them.
|
||||
|
||||
|
||||
Reference in new issue
Block a user