diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 67a60f47..fa117bc8 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -7,7 +7,7 @@ Steps, numbered as the run prints them: 1b provenance_report.py dump-catalog coverage, derivable.json 1c romset_recipes.py archive identification, reconstruction targets 2 refresh_data_dirs.py Dolphin Sys, PPSSPP assets and the other trees - 2a mame_hash_scraper MAME BIOS hashes from the driver source + 2a mame_hash_scraper MAME BIOS hashes, and device sets on a new release 2a2 fbneo_hash_scraper FBNeo BIOS hashes 2b check_buildbot_system.py buildbot freshness (--check-buildbot) 2b2 check_release_assets.py large-files release vs gitignored files diff --git a/scripts/scraper/mame_hash_scraper.py b/scripts/scraper/mame_hash_scraper.py index 3a5a850b..e3726fc5 100644 --- a/scripts/scraper/mame_hash_scraper.py +++ b/scripts/scraper/mame_hash_scraper.py @@ -334,6 +334,8 @@ def _run(args: argparse.Namespace) -> None: print(" no profiles with mamedev/mame upstream found") return + main_profile = _EMULATORS_DIR / "mame.yml" + pinned_before = _pinned_version(main_profile) for profile_path in profiles: is_main = profile_path.name == "mame.yml" diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame") @@ -356,6 +358,34 @@ def _run(args: argparse.Namespace) -> None: print() if args.dry_run: print("(dry run, no files modified)") + return + pinned_after = _pinned_version(main_profile) + if pinned_after != pinned_before: + # The device sets describe the release mame.yml is pinned on: left + # behind, the profile names two releases at once. + print(f"mame.yml moved from {pinned_before} to {pinned_after}: " + "merging the device ROM sets of that release") + _merge_devices(argparse.Namespace( + listxml=fetch_release_listxml(pinned_after), dry_run=False, + )) + + +def _pinned_version(profile_path: Path) -> str: + profile = yaml_load(profile_path.read_text(encoding="utf-8")) if profile_path.is_file() else {} + return str((profile or {}).get("core_version") or "") + + +def fetch_release_listxml(version: str) -> Path: + """The -listxml MAME publishes for a release, in the DAT importer's cache. + + 0.289 is the tag mame0289 and the asset mame0289lx.zip. + """ + from .romset_dat_importer import MAME_LISTXML_URL, _download + + tag = "mame" + version.replace(".", "") + return _download( + MAME_LISTXML_URL.format(tag=tag), _ROOT / ".cache" / "dats" / f"{tag}lx.zip" + ) def release_profile( diff --git a/tests/test_mame_devices.py b/tests/test_mame_devices.py index 12cc7069..b1542922 100644 --- a/tests/test_mame_devices.py +++ b/tests/test_mame_devices.py @@ -373,5 +373,48 @@ class Cli(unittest.TestCase): self.assertEqual(raised.exception.code, 2) +class TheDevicesFollowTheRelease(unittest.TestCase): + """Pipeline step 2a moved mame.yml to a new release and left the device + sets on the lines of the previous one: the profile named two releases.""" + + def _run(self, moves: bool, dry_run: bool = False) -> list: + import argparse + from unittest import mock + + from scripts.scraper import mame_hash_scraper as scraper + + with tempfile.TemporaryDirectory() as directory: + profile = Path(directory) / "mame.yml" + profile.write_text('core_version: "0.289"\nfiles: []\n') + + def merge(path, *_args, **_kwargs): + if moves: + profile.write_text('core_version: "0.290"\nfiles: []\n') + + merged = [] + with mock.patch.multiple( + scraper, + _EMULATORS_DIR=Path(directory), + _fetch_hashes=lambda force: {"bios_sets": {}, "version": "0.290"}, + _find_mame_profiles=lambda: [profile], + compute_diff=lambda *a, **k: {"updated": ["x"], "added": []}, + _format_diff=lambda *a, **k: [], + merge_mame_profile=merge, + fetch_release_listxml=lambda version: Path(f"mame{version}lx.zip"), + _merge_devices=merged.append, + ), mock.patch("sys.stdout", new=io.StringIO()): + scraper._run(argparse.Namespace(force=False, json=False, dry_run=dry_run)) + return merged + + def test_a_moved_pin_merges_that_release_devices(self): + (args,) = self._run(moves=True) + self.assertEqual(args.listxml, Path("mame0.290lx.zip")) + self.assertFalse(args.dry_run) + + def test_an_unmoved_pin_leaves_the_devices(self): + self.assertEqual(self._run(moves=False), []) + self.assertEqual(self._run(moves=True, dry_run=True), []) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 6fc7679e..9685ef25 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -854,6 +854,10 @@ go under the `Device ROM sets (generated from MAME -listxml)` comment at the end of `files:` with `required: false`. A set the profile already lists keeps every field a reader wrote and only gets its contents refreshed. +A plain run (pipeline step 2a) that moves `mame.yml` to a new release +downloads that release's `-listxml` into `.cache/dats/` and runs the device +merge itself, so the BIOS root sets and the device sets name one release. + ```bash python -m scripts.scraper.mame_hash_scraper --devices --listxml .cache/dats/mame0289lx.zip --dry-run python -m scripts.scraper.mame_hash_scraper --devices --listxml .cache/dats/mame0289lx.zip