fix: read fbneo console sets by driver name

In a console folder an FBNeo set is looked up under its driver name
(msx_1942), the bare name being the arcade set's. A run given its list as
a file reads the games index from beside it or the cache only when
--for-games was not asked. The installer keeps to Python 3.8 syntax, and
the testing guide names test_install_games.

install.ps1

@@ -16,7 +16,7 @@
     # and Invoke-WebRequest throws on a cut transfer before the hash is read:
     # the site's build rewrites this line there (scripts/sitebootstrap.py).
     $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
-    $defaultInstallSha256 = "8cf9ac83b620610cb469e65158e9e7f4efd0509242db86820403457c69ab3628"
+    $defaultInstallSha256 = "15763a260495b0b7c9cdfc43a36cf6aec50577bd423ae44976394552b13120db"
     $maximumInstallerBytes = 2MB
     $installerExit = 0
     $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -3950,7 +3950,9 @@ def _fetch_json(
         tries = attempts
         try:
             with urllib.request.urlopen(url, timeout=timeout) as resp:
-                return _read_limited_json(resp, limit, label, raw)
+                if raw:
+                    return _read_limited_json(resp, limit, label, raw=True)
+                return _read_limited_json(resp, limit, label)
         except urllib.error.HTTPError as exc:
             if exc.code < 500 and exc.code != 429 or attempt == attempts:
                 raise
@@ -10221,8 +10223,16 @@ def plan_games(plat: str, bios_path: Path, manifest: dict, folder: Path | None =
                 for device, members in (doc.get("devices") or {}).items():
                     for member in members:
                         devices.setdefault(member, []).append(device)
+                known = set(parents) | set(devices) | set(doc.get("no_bios") or [])
+                # FBNeo keys a console game by its driver (msx_1942): the
+                # MSX, NES and Arcade lists each hold a 1942.
+                consoles: dict[str, list[str]] = {}
+                for key in known:
+                    driver, cut, stem = key.partition("_")
+                    if cut and stem:
+                        consoles.setdefault(stem, []).append(driver)
                 arcades[name] = {
-                    "parents": parents, "devices": devices,
+                    "parents": parents, "devices": devices, "consoles": consoles,
                     "plain": set(doc.get("no_bios") or []),
                     "support": (set(doc.get("bios_sets") or []) | set(doc.get("bios") or {})
                                 | set(doc.get("devices") or {})),
@@ -10265,7 +10275,8 @@ def plan_games(plat: str, bios_path: Path, manifest: dict, folder: Path | None =
             facts = GameFacts(path, shown, game_folder)
             progress["done"] += 1
             if row.get("arcade"):
-                _read_arcade(facts, arcade_index(arcade) if arcade else None)
+                _read_arcade(facts, arcade_index(arcade) if arcade else None,
+                             _console_hints(game_folder))
             elif row_systems:
                 read_game(facts, row_systems[0], system_data(row_systems[0]), readers, cache)
             step()
@@ -10300,9 +10311,30 @@ def plan_games(plat: str, bios_path: Path, manifest: dict, folder: Path | None =
     return plan
 
 
-def _read_arcade(facts: GameFacts, index: dict | None) -> None:
+# Systems whose folder an arcade core reads as arcade sets, by their bare
+# set name. Any other system's sets are console games, keyed by driver.
+_ARCADE_SYSTEMS = frozenset({"arcade", "neogeo", "snk-neogeo", "sega-naomi", "sega-naomi2",
+                             "sammy-atomiswave", "sega-stv", "cps1", "cps2", "cps3"})
+
+
+def _console_hints(folder: GameFolder) -> tuple[str, ...] | None:
+    """None for an arcade folder, else the words of its systems and its
+    name, which tell one console's driver prefix from another's."""
+    systems = [str(s) for s in folder.row.get("systems") or []]
+    if not systems or any(s in _ARCADE_SYSTEMS or s.startswith("arcade") for s in systems):
+        return None
+    words = {word for system in systems for word in system.split("-")}
+    words.add(folder.path.name.lower())
+    return tuple(sorted(words))
+
+
+def _read_arcade(facts: GameFacts, index: dict | None,
+                 console: tuple[str, ...] | None = None) -> None:
     """An arcade set: a BIOS or device set (not a game), a game and the BIOS
-    sets its romof chain reaches, or a name the core's DAT does not hold."""
+    sets its romof chain reaches, or a name the core's DAT does not hold. In
+    a console's folder (console, the words of its systems) a set is looked
+    for under the core's driver name for it (msx_1942): the bare name is an
+    arcade set's."""
     if _suffix(facts.path.name) not in ARCADE_SUFFIXES:
         facts.kind = "support"
         return
@@ -10313,11 +10345,19 @@ def _read_arcade(facts: GameFacts, index: dict | None) -> None:
     if name in index["support"]:
         facts.kind = "support"
         return
-    facts.parents = tuple(sorted(index["parents"].get(name, ())))
-    facts.devices = tuple(sorted(index["devices"].get(name, ())))
-    if facts.parents or facts.devices or name in index["plain"]:
+    keys = [name]
+    if console is not None:
+        drivers = index.get("consoles", {}).get(name, [])
+        hinted = [d for d in drivers if d in console]
+        # A prefix no word of the folder names: every console's driver of
+        # that name, so that the right one is among them.
+        keys = [f"{d}_{name}" for d in sorted(hinted or drivers)] or [name]
+    parents = {p for key in keys for p in index["parents"].get(key, ())}
+    devices = {d for key in keys for d in index["devices"].get(key, ())}
+    facts.parents, facts.devices = tuple(sorted(parents)), tuple(sorted(devices))
+    if parents or devices or any(key in index["plain"] for key in keys):
         facts.recognised = True
-        facts.title = name
+        facts.title = keys[0] if len(keys) == 1 else name
     else:
         facts.reason = "unknown_set"
 
@@ -10555,7 +10595,8 @@ def read_games(args: argparse.Namespace, plat: str, bios_path: Path, manifest: d
     if folder is None and not asked and not games_probe(plat, bios_path, network,
                                                        os_type=os_type):
         return None
-    print(f"\nReading the games {'on the card' if folder is None else f'in {folder}'}...")
+    where = "on the card" if folder is None else f"in {folder}"
+    print(f"\nReading the games {where}...")
     try:
         plan = plan_games(plat, bios_path, manifest, folder, term, os_type=os_type,
                           network=network)

install.sh

@@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins
 # curl or wget stop on a cut transfer before the hash is read: the site's
 # build rewrites this line there (scripts/sitebootstrap.py).
 MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
-DEFAULT_INSTALL_SHA256="8cf9ac83b620610cb469e65158e9e7f4efd0509242db86820403457c69ab3628"
+DEFAULT_INSTALL_SHA256="15763a260495b0b7c9cdfc43a36cf6aec50577bd423ae44976394552b13120db"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

install/launchers/index.json

@@ -27,8 +27,8 @@
         {
           "dest": "roms/ports/retrobios/install.py",
           "repo_path": "install.py",
-          "size": 1077524,
-          "sha256": "8cf9ac83b620610cb469e65158e9e7f4efd0509242db86820403457c69ab3628",
+          "size": 1079771,
+          "sha256": "15763a260495b0b7c9cdfc43a36cf6aec50577bd423ae44976394552b13120db",
           "executable": false
         },
         {
@@ -71,8 +71,8 @@
     "muos": {
       "archive": {
         "dest": "ARCHIVE/RetroBIOS.muxapp",
-        "size": 1130701,
-        "sha256": "4381cfaa7fe2363671f334f1567e31e763df78d11fbadab3de135d7937878715"
+        "size": 1132948,
+        "sha256": "9915dccec657903ac3c4f70f4e2b71e4388e8554cb3c958a02962c2aaf603d56"
       },
       "members": [
         {
@@ -92,8 +92,8 @@
         {
           "name": "RetroBIOS/install.py",
           "repo_path": "install.py",
-          "size": 1077524,
-          "sha256": "8cf9ac83b620610cb469e65158e9e7f4efd0509242db86820403457c69ab3628",
+          "size": 1079771,
+          "sha256": "15763a260495b0b7c9cdfc43a36cf6aec50577bd423ae44976394552b13120db",
           "executable": false
         },
         {
@@ -166,8 +166,8 @@
         {
           "dest": ".config/modules/retrobios/install.py",
           "repo_path": "install.py",
-          "size": 1077524,
-          "sha256": "8cf9ac83b620610cb469e65158e9e7f4efd0509242db86820403457c69ab3628",
+          "size": 1079771,
+          "sha256": "15763a260495b0b7c9cdfc43a36cf6aec50577bd423ae44976394552b13120db",
           "executable": false
         },
         {

tests/test_install_games.py

@@ -66,6 +66,7 @@ FILES = {
     "neogeo.zip": (300, None, None, ["snk-neogeo"]),
     "fbneo/neogeo.zip": (300, None, ["fbneo"], None),
     "fbneo/pgm.zip": (200, None, ["fbneo"], None),
+    "fbneo/msx.zip": (150, None, ["fbneo"], None),
     "fbneo/samples/big.zip": (9000, None, ["fbneo"], None),
     "gba_bios.bin": (100, None, None, ["nintendo-gba"]),
 }
@@ -79,6 +80,7 @@ SETS = {
     "nintendo-gamecube|dolphin": (["dolphin/GC/USA/IPL.bin", "dolphin/GC/EUR/IPL.bin",
                                    "dolphin/GC/JAP/IPL.bin"], []),
     "arcade|fbneo": (["fbneo/samples/big.zip"], ["fbneo/samples/big.zip"]),
+    "msx|fbneo": (["fbneo/msx.zip"], []),
     "nintendo-gba|mgba": (["gba_bios.bin"], ["gba_bios.bin"]),
     "nintendo-gba|gpsp": (["gba_bios.bin"], []),
 }
@@ -153,9 +155,10 @@ SYSTEMS = {
 }
 
 ARCADE = {
-    "bios": {"neogeo": ["mslug", "mslug2"], "pgm": ["orlegend"], "nobios": ["lost"]},
+    "bios": {"neogeo": ["mslug", "mslug2"], "pgm": ["orlegend"], "nobios": ["lost"],
+             "msx": ["msx_1942"]},
     "devices": {}, "chd": [], "missing_parents": {},
-    "no_bios": ["pacman"], "bios_sets": ["neogeo", "pgm", "nobios"],
+    "no_bios": ["1942", "nes_1942", "pacman"], "bios_sets": ["msx", "neogeo", "nobios", "pgm"],
 }
 
 SETUP = {
@@ -179,6 +182,9 @@ SETUP = {
         {"folder": "FBNEO", "systems": ["arcade"], "basis": "id", "profile": "fbneo",
          "arcade": "fbneo", "extensions": [".zip", ".7z"], "fallback": ["arcade|fbneo"],
          "source_ref": "x"},
+        {"folder": "MSX", "systems": ["msx"], "basis": "id", "profile": "fbneo",
+         "arcade": "fbneo", "extensions": [".zip"], "fallback": ["msx|fbneo"],
+         "source_ref": "x"},
         {"folder": "GBA", "systems": ["nintendo-gba"], "basis": "id", "profile": "mgba",
          "extensions": [".gba"], "fallback": ["nintendo-gba|mgba"],
          "board_defaults": {"rpi": {"core": "gpsp", "profile": "gpsp",
@@ -615,10 +621,28 @@ class Arcade(GamesTestCase):
         self.card.game("FBNEO", "zzzz.zip", b"PK")
         plan = self.plan()
         self.assertEqual(self.kept(plan),
-                         ["fbneo/neogeo.zip", "fbneo/pgm.zip", "fbneo/samples/big.zip"])
+                         ["fbneo/msx.zip", "fbneo/neogeo.zip", "fbneo/pgm.zip",
+                          "fbneo/samples/big.zip"])
         self.assertEqual([(g.shown, g.reason) for g in plan.fallback],
                          [("FBNEO/zzzz.zip", "unknown_set")])
 
+    def test_a_console_set_is_read_under_its_driver_name(self):
+        # msx_1942 loads the MSX BIOS, the arcade 1942 needs none.
+        self.card.game("MSX", "1942.zip", b"PK")
+        plan = self.plan()
+        self.assertEqual(self.kept(plan), ["fbneo/msx.zip"])
+        self.assertEqual(plan.games[0].title, "msx_1942")
+
+    def test_an_arcade_folder_reads_the_bare_set_name(self):
+        self.card.game("FBNEO", "1942.zip", b"PK")
+        plan = self.plan()
+        self.assertEqual(self.kept(plan), [])
+        self.assertTrue(plan.games[0].recognised)
+
+    def test_a_console_bios_on_the_card_is_not_a_game(self):
+        self.card.game("MSX", "msx.zip", b"PK")
+        self.assertEqual(self.plan().support, 1)
+
     def test_a_parent_the_list_does_not_hold_falls_back(self):
         self.card.game("FBNEO", "lost.zip", b"PK")
         plan = self.plan()

wiki/testing-guide.md

@@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards.
 | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
 | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
 | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
-| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
+| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
 | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` |
 | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |
 | Measurement records | synthetic record files, the committed `measurements/` against HEAD, the **Proof** line naming the BIOS a core ran, and that no page shows a Tested block, a region outcome or a time per frame | `test_measurements` |