chore: merge main into the Starlight branch

tests/test_archive_recipes.py

@@ -20,9 +20,12 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
 
 # Sets whose archive also carries the ROMs of devices the machine references
-# (listxml device_ref): the NAOMI boards pull the JVS I/O board and the Maple
-# interface, and MAME finds those ROMs in the machine's own archive too.
+# (listxml device_ref). A device looks for its ROMs under its own name, then
+# under the system's (device_t::searchpath, src/emu/device.cpp:140-152), so
+# MAME finds them in the machine's own archive too. A device set is not a
+# system: the devices it holds are never looked for in its archive.
 DEVICE_ROMS = {
+    # NAOMI boards: the JVS I/O board and the Maple interface
     "airlbios.zip": ("jvs13551.zip", "mie.zip"),
     "f355bios.zip": ("jvs13551.zip", "mie.zip"),
     "f355dlx.zip": ("jvs13551.zip", "mie.zip"),
@@ -30,18 +33,41 @@ DEVICE_ROMS = {
     "naomi.zip": ("jvs13551.zip", "mie.zip"),
     "naomi2.zip": ("jvs13551.zip", "mie.zip"),
     "naomigd.zip": ("jvs13551.zip", "mie.zip"),
+    # Teletext character ROM: SAA5050 in bbcmet (acorn/bbcm.cpp:701), which
+    # bbcm builds on (line 764)
+    "bbcm.zip": ("saa5050.zip",),
+    # DragonDOS, the default cartridge (trs/dragon.cpp:544, 562)
+    "dragon32.zip": ("dragon_fdc.zip",),
+    "dragon64.zip": ("dragon_fdc.zip",),
+    # PC-80S31 disk unit (nec/pc6001.cpp:1644 through pc6601_fdc_config,
+    # nec/pc8001.cpp:1257)
+    "pc6001mk2sr.zip": ("pc80s31.zip",),
+    "pc8001mk2sr.zip": ("pc80s31.zip",),
 }
 
 # Archives that share a set name without being that set.
 NOT_ROMSETS = {"samples", "artwork"}
 
-# The defect added whole slot-option trees, dozens of members at a time.
-MIN_EXTRA = 3
+MIN_EXTRA = 1
 
 # Kept as they are: rewriting them would drop bytes the collection holds
-# nowhere else, and no recipe names the set those bytes belong to.
+# nowhere else, or they are another emulator's archive under a MAME set name.
 KEPT = {
-    "bios/Acorn/BBC Micro/.variants/bbcb.zip": "zdfs-0.90.rom is held only here",
+    "bios/Acorn/BBC Micro/.variants/bbcb.zip": (
+        "MAME 0.150's bbcb loads its DFS ROMs from the machine archive "
+        "(src/mess/drivers/bbc.c ROM_START(bbcb)), older than every imported "
+        "listxml; zdfs-0.90.rom is held only here"
+    ),
+    "bios/Arcade/Arcade/.variants/cvs.zip.3c69f366": (
+        "the 2001 mame.dk distribution of cvs.zip; its cvs.txt is held only here"
+    ),
+    "bios/Mattel/Aquarius/.variants/aquarius_ar.zip.73026a45": (
+        "a platform pack copy; its TMCTV.png is held only here"
+    ),
+    "bios/Sega/Dreamcast/dc.zip": (
+        "RetroBat's dc/dc.zip holds dc_boot.bin and dc_flash.bin, the names the "
+        "Dreamcast BIOS goes by outside MAME, not MAME's dc set"
+    ),
 }
 
 

tests/test_cache_independent_counts.py

@@ -0,0 +1,63 @@
+"""A platform's coverage does not depend on which data caches are on disk.
+
+EmuELEC declares 19 files of the Tyrian 2.1 distribution that bios/ did not
+hold (setup.ini, netterm.int...). They resolved only through data/opentyrian,
+the cache refresh_data_dirs.py fills from a release asset, so verify gave
+544/572 in a checkout with that cache and 525/572 in a fresh worktree, for the
+same commit. A file a platform declares is collected in bios/; a data
+directory cache is not a collection.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+
+class DeclaredFilesResolveWithoutDataCaches(unittest.TestCase):
+    def test_no_platform_entry_rests_on_a_data_directory_cache(self):
+        from common import (  # noqa: PLC0415
+            build_zip_contents_index,
+            list_registered_platforms,
+            load_data_dir_registry,
+            load_database,
+            load_platform_config,
+        )
+        from packresolve import resolve_declaration  # noqa: PLC0415
+
+        platforms_dir = REPO_ROOT / "platforms"
+        registry = load_data_dir_registry(str(platforms_dir))
+        caches = [
+            entry["local_cache"] for entry in registry.values()
+            if (REPO_ROOT / entry.get("local_cache", "")).is_dir()
+        ]
+        if not caches or not (REPO_ROOT / "database.json").is_file():
+            self.skipTest("no data directory cache on disk (refresh_data_dirs.py fills them)")
+
+        cwd = os.getcwd()
+        os.chdir(REPO_ROOT)
+        try:
+            db = load_database("database.json")
+            zip_contents = build_zip_contents_index(db)
+            offenders = []
+            for platform in list_registered_platforms(str(platforms_dir), include_archived=True):
+                config = load_platform_config(platform, str(platforms_dir))
+                for sys_id, system in config.get("systems", {}).items():
+                    for entry in system.get("files", []) or []:
+                        path, status = resolve_declaration(
+                            entry, db, "bios", zip_contents, registry, None, offline=True,
+                        )
+                        if status.startswith("data_dir"):
+                            offenders.append(f"{platform} {sys_id} {entry.get('name')}: {path}")
+        finally:
+            os.chdir(cwd)
+        self.assertEqual(offenders, [], "\n".join(offenders))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_dedup.py

@@ -341,5 +341,31 @@ class OnlyFilesSharingASizeAreHashed(_Bios):
                          sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin")))
 
 
+
+class CloneChainsCollapse(unittest.TestCase):
+    """A later run removed sys68k_cpu21.zip as a copy of sys68k_cpu20.zip
+    while the map still named it canonical for five clones: those clones
+    pointed at a file that no longer existed."""
+
+    def test_clones_follow_to_the_canonical_that_exists(self):
+        import dedup  # noqa: PLC0415
+
+        chain = {
+            "cpu20.zip": {"clones": ["cpu21.zip"], "sha1": "a", "total_copies": 1},
+            "cpu21.zip": {"clones": ["cpu21a.zip", "cpu21b.zip"], "sha1": "a", "total_copies": 2},
+        }
+        self.assertEqual(dedup.collapse_clone_chains(chain), {
+            "cpu20.zip": {"clones": ["cpu21.zip", "cpu21a.zip", "cpu21b.zip"],
+                          "sha1": "a", "total_copies": 3},
+        })
+
+    def test_a_loop_is_refused(self):
+        import dedup  # noqa: PLC0415
+
+        loop = {"a.zip": {"clones": ["b.zip"]}, "b.zip": {"clones": ["a.zip"]}}
+        with self.assertRaises(ValueError):
+            dedup.collapse_clone_chains(loop)
+
+
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

tests/test_extract_targets.py

@@ -67,9 +67,14 @@ class ThePackGuideNamesTheInstallerDefault(unittest.TestCase):
             rule["path"] for rule in registry["retrobat"]["install"]["detect"]
             if rule["os"] == "windows" and rule["method"] == "path_exists"
         )
-        guide = (REPO_ROOT / "scripts" / "packreadme.py").read_text(encoding="utf-8")
+        sys.path.insert(0, str(REPO_ROOT / "scripts"))
+        from siteguides import pack_manual  # noqa: PLC0415
+
+        # The pack guide text is data (platforms/_guides.yml): read what the
+        # pack README prints, not the source of the script that prints it.
+        guide = pack_manual("retrobat")
         self.assertEqual(first, "C:\\RetroBat\\bios")
-        self.assertIn("(default: C:\\\\RetroBat\\\\bios\\\\)", guide)
+        self.assertIn("(default: C:\\RetroBat\\bios\\)", guide)
 
 
 if __name__ == "__main__":

tests/test_mame_ref_audit.py

@@ -117,9 +117,14 @@ class TestCorpusStaysCorrect(unittest.TestCase):
             self.skipTest("no upstream cache")
         for name in ("mame", "mamearcade", "mamemess", "groovymame"):
             with self.subTest(emulator=name):
-                findings, agreed, _ = mame_ref_audit.audit(
-                    name, str(ROOT / "emulators"), str(cache), True
-                )
+                try:
+                    findings, agreed, _ = mame_ref_audit.audit(
+                        name, str(ROOT / "emulators"), str(cache), True
+                    )
+                except mame_ref_audit.upstream.NotCachedError as exc:
+                    # A cache that holds some drivers and not others is as
+                    # cold as an empty one for this profile.
+                    self.skipTest(f"{name}: {exc}")
                 if not agreed and not findings:
                     self.skipTest(f"{name}: driver sources not cached")
                 self.assertEqual(

tests/test_schema_semantic_gate.py

@@ -0,0 +1,98 @@
+"""Schema-invalid synthetic manifests must not crash semantic validation."""
+
+from __future__ import annotations
+
+import io
+import json
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+try:
+    import validate_schemas
+except ImportError as exc:
+    validate_schemas = None
+    IMPORT_ERROR = str(exc)
+else:
+    IMPORT_ERROR = ""
+
+
+@unittest.skipIf(validate_schemas is None, f"validate_schemas unavailable: {IMPORT_ERROR}")
+class SemanticManifestValidation(unittest.TestCase):
+    def test_invalid_synthetic_manifest_reports_schema_errors_without_crashing(self):
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT) as directory:
+            root = Path(directory)
+            (root / "emulators").mkdir()
+            platforms = root / "platforms"
+            platforms.mkdir()
+            (platforms / "_data_dirs.yml").write_text(
+                "data_directories: {}\n", encoding="utf-8"
+            )
+            (root / "database.json").write_text(
+                json.dumps({"files": None}), encoding="utf-8"
+            )
+            install = root / "install"
+            install.mkdir()
+            (install / "synthetic.json").write_text(
+                json.dumps({"files": None}), encoding="utf-8"
+            )
+            api = root / "docs" / "api" / "v1"
+            api.mkdir(parents=True)
+            (api / "catalog.json").write_text(
+                json.dumps({"count": 1, "items": None}), encoding="utf-8"
+            )
+
+            output = io.StringIO()
+            with mock.patch.object(validate_schemas, "ROOT", root), mock.patch.object(
+                sys, "argv", ["validate_schemas.py"]
+            ), redirect_stdout(output):
+                result = validate_schemas.main()
+
+        self.assertEqual(result, 1)
+        text = output.getvalue().replace("\\", "/")
+        self.assertIn("database.json", text)
+        self.assertIn("install/synthetic.json", text)
+        self.assertIn("docs/api/v1/catalog.json", text)
+        self.assertIn("files", text)
+        self.assertIn("items", text)
+        self.assertNotIn("Traceback", text)
+
+    def test_valid_synthetic_manifest_still_runs_semantic_checks(self):
+        manifest = {
+            "manifest_version": 2,
+            "source": "full",
+            "regions": [],
+            "platform": "synthetic",
+            "display_name": "Synthetic",
+            "version": "test",
+            "generated": "2026-10-21T00:00:00Z",
+            "base_destination": "",
+            "detect": [],
+            "standalone_copies": [],
+            "total_files": 1,
+            "total_size": 0,
+            "total_omitted": 0,
+            "omitted_files": [],
+            "files": [],
+        }
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT) as directory:
+            path = Path(directory) / "synthetic.json"
+            path.write_text(json.dumps(manifest), encoding="utf-8")
+            errors = validate_schemas._validate_json_files(
+                [path],
+                "install-manifest.schema.json",
+                validate_schemas._semantic_install_checks,
+            )
+
+        self.assertEqual(len(errors), 1)
+        self.assertIn("total_files does not equal len(files)", errors[0])
+
+
+if __name__ == "__main__":
+    unittest.main()