fix: merge mame devices when the pin moves

Step 2a advanced mame.yml's core_version and upstream_commit to a new
release and left the device ROM sets on the previous one. A plain run
that moves the pin now fetches that release's listxml into .cache/dats
and runs the device merge, which checks version and commit itself.

scripts/pipeline.py

@@ -7,7 +7,7 @@ Steps, numbered as the run prints them:
   1b   provenance_report.py         dump-catalog coverage, derivable.json
   1c   romset_recipes.py            archive identification, reconstruction targets
   2    refresh_data_dirs.py         Dolphin Sys, PPSSPP assets and the other trees
-  2a   mame_hash_scraper            MAME BIOS hashes from the driver source
+  2a   mame_hash_scraper            MAME BIOS hashes, and device sets on a new release
   2a2  fbneo_hash_scraper           FBNeo BIOS hashes
   2b   check_buildbot_system.py     buildbot freshness (--check-buildbot)
   2b2  check_release_assets.py      large-files release vs gitignored files

scripts/scraper/mame_hash_scraper.py

@@ -334,6 +334,8 @@ def _run(args: argparse.Namespace) -> None:
         print("  no profiles with mamedev/mame upstream found")
         return
 
+    main_profile = _EMULATORS_DIR / "mame.yml"
+    pinned_before = _pinned_version(main_profile)
     for profile_path in profiles:
         is_main = profile_path.name == "mame.yml"
         diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame")
@@ -356,6 +358,34 @@ def _run(args: argparse.Namespace) -> None:
     print()
     if args.dry_run:
         print("(dry run, no files modified)")
+        return
+    pinned_after = _pinned_version(main_profile)
+    if pinned_after != pinned_before:
+        # The device sets describe the release mame.yml is pinned on: left
+        # behind, the profile names two releases at once.
+        print(f"mame.yml moved from {pinned_before} to {pinned_after}: "
+              "merging the device ROM sets of that release")
+        _merge_devices(argparse.Namespace(
+            listxml=fetch_release_listxml(pinned_after), dry_run=False,
+        ))
+
+
+def _pinned_version(profile_path: Path) -> str:
+    profile = yaml_load(profile_path.read_text(encoding="utf-8")) if profile_path.is_file() else {}
+    return str((profile or {}).get("core_version") or "")
+
+
+def fetch_release_listxml(version: str) -> Path:
+    """The -listxml MAME publishes for a release, in the DAT importer's cache.
+
+    0.289 is the tag mame0289 and the asset mame0289lx.zip.
+    """
+    from .romset_dat_importer import MAME_LISTXML_URL, _download
+
+    tag = "mame" + version.replace(".", "")
+    return _download(
+        MAME_LISTXML_URL.format(tag=tag), _ROOT / ".cache" / "dats" / f"{tag}lx.zip"
+    )
 
 
 def release_profile(

tests/test_mame_devices.py

@@ -373,5 +373,48 @@ class Cli(unittest.TestCase):
         self.assertEqual(raised.exception.code, 2)
 
 
+class TheDevicesFollowTheRelease(unittest.TestCase):
+    """Pipeline step 2a moved mame.yml to a new release and left the device
+    sets on the lines of the previous one: the profile named two releases."""
+
+    def _run(self, moves: bool, dry_run: bool = False) -> list:
+        import argparse
+        from unittest import mock
+
+        from scripts.scraper import mame_hash_scraper as scraper
+
+        with tempfile.TemporaryDirectory() as directory:
+            profile = Path(directory) / "mame.yml"
+            profile.write_text('core_version: "0.289"\nfiles: []\n')
+
+            def merge(path, *_args, **_kwargs):
+                if moves:
+                    profile.write_text('core_version: "0.290"\nfiles: []\n')
+
+            merged = []
+            with mock.patch.multiple(
+                scraper,
+                _EMULATORS_DIR=Path(directory),
+                _fetch_hashes=lambda force: {"bios_sets": {}, "version": "0.290"},
+                _find_mame_profiles=lambda: [profile],
+                compute_diff=lambda *a, **k: {"updated": ["x"], "added": []},
+                _format_diff=lambda *a, **k: [],
+                merge_mame_profile=merge,
+                fetch_release_listxml=lambda version: Path(f"mame{version}lx.zip"),
+                _merge_devices=merged.append,
+            ), mock.patch("sys.stdout", new=io.StringIO()):
+                scraper._run(argparse.Namespace(force=False, json=False, dry_run=dry_run))
+            return merged
+
+    def test_a_moved_pin_merges_that_release_devices(self):
+        (args,) = self._run(moves=True)
+        self.assertEqual(args.listxml, Path("mame0.290lx.zip"))
+        self.assertFalse(args.dry_run)
+
+    def test_an_unmoved_pin_leaves_the_devices(self):
+        self.assertEqual(self._run(moves=False), [])
+        self.assertEqual(self._run(moves=True, dry_run=True), [])
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/tools.md

@@ -854,6 +854,10 @@ go under the `Device ROM sets (generated from MAME -listxml)` comment at the end
 of `files:` with `required: false`. A set the profile already lists keeps every
 field a reader wrote and only gets its contents refreshed.
 
+A plain run (pipeline step 2a) that moves `mame.yml` to a new release
+downloads that release's `-listxml` into `.cache/dats/` and runs the device
+merge itself, so the BIOS root sets and the device sets name one release.
+
 ```bash
 python -m scripts.scraper.mame_hash_scraper --devices --listxml .cache/dats/mame0289lx.zip --dry-run
 python -m scripts.scraper.mame_hash_scraper --devices --listxml .cache/dats/mame0289lx.zip