chore: merge redump, mame and fbneo data fixes

scripts/common.py

@@ -2663,6 +2663,20 @@ SYSTEM_ALIASES: dict[str, str] = {
     # profiles use only through these.
     "sega-segacd": "sega-megacd",
     "nes-fds": "nintendo-fds",
+    # One machine, two spellings across platform folders and profiles:
+    # Recalbox's apple2gs, Batocera's astrocade, KNULLI and RetroBat's
+    # astrocde, MiSTer's creativision, Batocera's gamepocketcomputer
+    # folder, BizHawk's Super Game Boy, the FBA 2012 board names.
+    "apple2gs": "apple-iigs",
+    "astrocade": "bally-astrocade",
+    "astrocde": "bally-astrocade",
+    "creativision": "crvision",
+    "gamepocketcomputer": "gamepock",
+    "nintendo-super-game-boy": "nintendo-sgb",
+    "cps1": "capcom-cps1",
+    "cps2": "capcom-cps2",
+    "cps3": "capcom-cps3",
+    "pgm": "igs-pgm",
 }
 
 

scripts/packextras.py

@@ -1377,8 +1377,17 @@ class _ClaimRanks:
 
 
 def _names_content(entry: dict) -> bool:
-    """Whether a report entry declares the bytes it wants, not only a name."""
-    return any(entry.get(field) for field in ("sha1", "md5", "sha256", "crc32"))
+    """Whether a report entry declares the bytes it wants, not only a name.
+
+    The claims folded onto the entry count: bsnes asks for SGB1.sfc by name
+    and bsnes-hd pins its md5 under the same system, and the entry carrying
+    both went after nSide's SGB1.sfc/ tree, which took the path from the file.
+    """
+    return any(
+        claim.get(field)
+        for claim in (entry, *entry.get("other_claims", []))
+        for field in ("sha1", "md5", "sha256", "crc32")
+    )
 
 
 def _save_fields(extra: dict, pack_path: str, placement: ExtraPlacement) -> None:

scripts/scraper/mame_devices.py

@@ -272,6 +272,33 @@ def attribute_systems(
     return {name: sorted(found) for name, found in systems.items()}
 
 
+def device_systems(
+    machines: dict[str, MachineLinks],
+    seeds: dict[str, str],
+    fallback: str,
+) -> dict[str, list[str]]:
+    """Systems whose machines can load each set, unplaced machines under *fallback*.
+
+    attribute_systems files a set under the systems of the machines a seed
+    places. A machine no seed places (a computer no platform runs through
+    this profile, a game outside every BIOS family) still loads its devices,
+    and MAME starts it from its set name, the way an arcade folder passes a
+    game: it counts as a machine of *fallback*. A set only such machines
+    load (an Amiga keyboard, an ISA card) then goes under *fallback* alone,
+    and a set both kinds load (ym2413, for the CoCo and for arcade games)
+    under both. A set no runnable machine of the release instantiates keeps
+    no system: the document never names its machine.
+    """
+    if not fallback:
+        return attribute_systems(machines, seeds)
+    unplaced = {
+        name: fallback
+        for name, links in machines.items()
+        if not links.isdevice and not _seed_system(name, machines, seeds)
+    }
+    return attribute_systems(machines, {**seeds, **unplaced})
+
+
 def platform_seeds(
     stated: Iterable[tuple[str, str]],
     machines: dict[str, MachineLinks],

scripts/scraper/mame_hash_scraper.py

@@ -48,6 +48,7 @@ from .mame_devices import (
     attribute_softlists,
     attribute_systems,
     default_floppy_samples,
+    device_systems,
     drive_form_factor,
     floppy_drives,
     floppy_sample_systems,
@@ -70,6 +71,9 @@ _CLONE_DIR = _ROOT / "tmp" / f"mame-{os.getpid()}"
 _EMULATORS_DIR = _ROOT / "emulators"
 _PLATFORMS_DIR = _ROOT / "platforms"
 _FLOPPY_SOURCE = "src/devices/imagedev/floppy.cpp"
+# Where a device set goes when only machines no seed places load it: MAME
+# starts such a machine from its set name, as an arcade folder passes one.
+UNPLACED_SYSTEM = "arcade"
 _REPO_URL = "https://github.com/mamedev/mame.git"
 _STALE_HOURS = 24
 
@@ -570,9 +574,14 @@ def _merge_devices(args: argparse.Namespace) -> None:
     stated, conflicts = platform_seeds(
         _platform_archives(), machines, profile.get("systems") or [], written
     )
-    systems = attribute_systems(machines, {**stated, **written})
+    seeds = {**stated, **written}
+    systems = attribute_systems(machines, seeds)
+    listed = profile.get("systems") or []
+    filed = device_systems(
+        machines, seeds, UNPLACED_SYSTEM if UNPLACED_SYSTEM in listed else ""
+    )
     for set_name, device in devices.items():
-        device["systems"] = systems.get(set_name, [])
+        device["systems"] = filed.get(set_name, [])
     report = merge_mame_devices(str(profile_path), devices, write=not args.dry_run)
 
     shipped = upstream.list_directory(repo, commit, "hash", upstream.CACHE_DIR)
@@ -602,10 +611,12 @@ def _merge_devices(args: argparse.Namespace) -> None:
     for set_name in report["conflict"]:
         print(f"    ! {set_name}.zip is listed under another category, left as written")
     by_count = Counter(len(device["systems"]) for device in devices.values())
+    fallen = sum(1 for device in devices.values() if UNPLACED_SYSTEM in device["systems"])
     print(
         f"  systems: {by_count[1]} sets under one system, "
         f"{sum(n for k, n in by_count.items() if k > 1)} under several, "
-        f"{by_count[0]} loaded by no placed machine "
+        f"{fallen} of them under {UNPLACED_SYSTEM} (loaded by machines no entry places), "
+        f"{by_count[0]} instantiated by no machine of the release "
         "(they serve every system of the profile)"
     )
     print(

scripts/scraper/redump_dat_scraper.py

@@ -1,9 +1,11 @@
 """Fetch Redump BIOS DAT files and write a provenance snapshot.
 
-Redump serves BIOS DATs as static clrmamepro files linked from
-https://redump.info/downloads under /static/bios/. The downloads page
-is scanned for those links, so new BIOS DATs are picked up without
-code changes.
+Redump serves BIOS DATs as Logiqx files linked from
+https://redump.info/downloads. The links were static files under
+/static/bios/<name>.dat and are now one route per system,
+/bios/<CODE> (PSX, PS2, GC, XBOX), served as an attachment. The
+downloads page is scanned for either shape, so new BIOS DATs are
+picked up without code changes.
 
 Run manually, then commit the snapshot:
     python -m scripts.scraper.redump_dat_scraper --dry-run
@@ -40,6 +42,7 @@ def _fetch(url: str) -> str:
 def discover_bios_datfiles(downloads_html: str) -> list[str]:
     """Extract BIOS DAT paths from the downloads page."""
     paths = set(re.findall(r'href="(/static/bios/[^"]+\.dat)"', downloads_html))
+    paths.update(re.findall(r'href="(/bios/[A-Za-z0-9_-]+)"', downloads_html))
     return sorted(paths)
 
 

scripts/siterender.py

@@ -860,7 +860,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "apple-iie": "Apple IIe",
     "apple-ios": "Apple iOS",
     "apple2": "Apple II",
-    "apple2gs": "Apple IIGS",
     "arcade": "Arcade games",
     "arcade-laserdisc": "Laserdisc arcade games",
     "arcadia-systems": "Arcadia Systems Multi Select",
@@ -869,7 +868,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "aristocrat-mk6": "Aristocrat MK6",
     "aristocrat-mk7": "Aristocrat MK7",
     "arm": "ARM",
-    "astrocde": "Bally Astrocade",
     "atari-falcon": "Atari Falcon",
     "atari-jaguarcd": "Atari Jaguar CD",
     "atari-ste": "Atari STE",
@@ -920,10 +918,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "commodore-cdtv": "Commodore CDTV",
     "commodore-cubo": "CD Express Cubo CD32",
     "coolbar": "Coolbar feature phones",
-    "cps1": "Capcom CPS-1",
-    "cps2": "Capcom CPS-2",
-    "cps3": "Capcom CPS-3",
-    "crvision": "VTech CreatiVision",
     "cstrike": "Counter-Strike",
     "ctf": "Quake II: Capture the Flag",
     "czero": "Counter-Strike: Condition Zero",
@@ -1073,7 +1067,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "nintendo-nss": "Nintendo Super System",
     "nintendo-playch10": "Nintendo PlayChoice-10",
     "nintendo-sfcbox": "Nintendo Super Famicom Box",
-    "nintendo-sgb": "Nintendo Super Game Boy",
     "nintendo-virtualboy": "Nintendo Virtual Boy",
     "nintendo-wii-arcade": "Nintendo Wii arcade boards",
     "nmk16": "NMK16",
@@ -1092,7 +1085,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "pce-tourvision": "TourVision",
     "pdark": "Perfect Dark",
     "pegasus": "Aamber Pegasus",
-    "pgm": "IGS PolyGame Master",
     "philips-p2000t": "Philips P2000T",
     "philips-vg5k": "Philips VG 5000",
     "picocomputer-6502": "Picocomputer 6502 programs",
@@ -1216,6 +1208,7 @@ SYSTEM_NAMES: dict[str, str] = {
     "tomb-raider-1": "Tomb Raider",
     "tomb-raider-2": "Tomb Raider II",
     "transport-tycoon-deluxe": "Transport Tycoon Deluxe",
+    "tvgames": "Plug-and-play TV games",
     "tyrian": "Tyrian",
     "ur-quan-masters": "Star Control II",
     "urban-terror": "Urban Terror",