chore: merge audit batch, resolver and profiles

tests/test_audit_regressions.py

@@ -5713,6 +5713,10 @@ class AHashScanLabelNamesNoOtherDump(unittest.TestCase):
             "beetle_psx reads its first 512 KB under the same name",
         ("duckstation-legacy", "ps1_rom.bin"):
             "the same whole PS3 image, whose first 512 KB src/core/bios.cpp:89 hashes",
+        ("openmsx", "COLECO.ROM"):
+            "share/machines/ColecoVision_SGM.xml:56 names it, tried beside the machine XML",
+        ("openmsx", "fmpac.rom"):
+            "share/machines/Boosted_MSX2_EN.xml:83 names it, tried beside the machine XML",
     }
 
     @staticmethod
@@ -6821,5 +6825,128 @@ class PipelineChildrenSkipNodeCompileCache(unittest.TestCase):
         self.assertIn('"NODE_DISABLE_COMPILE_CACHE": "1"', source)
 
 
+class HiganSuperGameBoyFolderIsFree(unittest.TestCase):
+    """higan reads program.rom in the option's folder, SGB.sfc/, then Super
+    GameBoy.sfc/. bsnes, Mesen-S and ares read SGB1.sfc and SGB2.sfc as files
+    in the same system directory, and SuperSnes9x sgb.sfc, so a pack that only
+    had the first two folders never gave the higan cores a Super Game Boy."""
+
+    def test_each_higan_core_has_a_folder_no_core_reads_as_a_file(self):
+        import yaml
+
+        file_names: set[str] = set()
+        for path in (ROOT / "emulators").glob("*.yml"):
+            document = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            for entry in document.get("files") or []:
+                for name in [entry.get("name"), *(entry.get("aliases") or [])]:
+                    if name:
+                        file_names.add(str(name).casefold())
+                if entry.get("path"):
+                    file_names.add(str(entry["path"]).casefold())
+        for profile in ("higan_sfc", "higan_sfc_balanced"):
+            document = yaml.safe_load(
+                (ROOT / "emulators" / f"{profile}.yml").read_text(encoding="utf-8")
+            )
+            folders = {
+                str(entry["path"]).split("/", 1)[0]
+                for entry in document["files"]
+                if str(entry.get("path") or "").endswith("/program.rom")
+                and entry.get("system") == "nintendo-sgb"
+            }
+            with self.subTest(profile=profile):
+                self.assertTrue(
+                    [f for f in folders if f.casefold() not in file_names], folders
+                )
+
+
+class ProfileNotesDescribeTheCoreAlone(unittest.TestCase):
+    """mednafen_pce_fast, sdlpop and x64sdl described the core by what
+    another profile of this directory does ("unlike beetle_pce", "unlike the
+    other engines in this directory"), which means nothing once the profile
+    is read on its own, and 21 lines shouted NOT, ALL or MUST."""
+
+    SHOUT = re.compile(r"\b(NOT|NEVER|ONLY|MUST|ALL|DIFFERENT)\b(?! FOUND)")
+
+    def test_no_profile_shouts_or_compares_with_another_profile(self):
+        names = {path.stem.casefold() for path in (ROOT / "emulators").glob("*.yml")}
+        names |= {name.replace("_", "-") for name in names}
+        found = []
+        for path in sorted((ROOT / "emulators").glob("*.yml")):
+            text = path.read_text(encoding="utf-8")
+            for number, line in enumerate(text.splitlines(), 1):
+                if self.SHOUT.search(line):
+                    found.append(f"{path.stem}:{number}")
+                for match in re.finditer(r"\bunlike (?:the )?([\w+-]+)", line, re.I):
+                    if match.group(1).casefold() in names:
+                        found.append(f"{path.stem}:{number}")
+        self.assertEqual(found, [])
+
+
+class Ps2BiosImagesAreFoundByContent(unittest.TestCase):
+    """PCSX2 and its forks scan the BIOS folder and keep the first 4 to 8 MB
+    image with a romdir (pcsx2/ps2/BiosTools.cpp:258-282 at f0523f8a). With
+    no found_by the label ps2-0230a-20080220.bin read as a name the core
+    opens, and the RetroDECK export added it as a required file."""
+
+    def test_every_ps2_image_scan_is_marked(self):
+        import yaml
+
+        from validation import opens_by_name
+
+        named = []
+        for path in sorted((ROOT / "emulators").glob("*.yml")):
+            profile = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            if profile.get("bios_mode") != "agnostic":
+                continue
+            for entry in profile.get("files") or []:
+                if (
+                    entry.get("min_size") == 4194304
+                    and entry.get("max_size") == 8388608
+                    and opens_by_name(entry, profile)
+                ):
+                    named.append(f"{path.stem}: {entry['name']}")
+        self.assertEqual(named, [])
+
+
+class Dev9FilesAreReadFromTheWorkingDirectory(unittest.TestCase):
+    """DEV9 opens eeprom.dat and flash.dat by bare name (PCSX2
+    pcsx2/DEV9/DEV9.cpp:144, AetherSX2 DEV9/flash.cpp:89): the working
+    directory, never the BIOS folder. Eight profiles left load_from unset,
+    so the file was packed into the BIOS folder, where nothing reads it."""
+
+    def test_every_dev9_file_names_the_working_directory(self):
+        import yaml
+
+        unset = []
+        for path in sorted((ROOT / "emulators").glob("*.yml")):
+            profile = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            for entry in profile.get("files") or []:
+                if entry.get("name") in ("eeprom.dat", "flash.dat") and "DEV9" in str(
+                    entry.get("source_ref")
+                ) and entry.get("load_from") != "working_directory":
+                    unset.append(f"{path.stem}: {entry['name']}")
+        self.assertEqual(unset, [])
+
+
+class OpenMsxSystemRomsAreFoundBySha1(unittest.TestCase):
+    """openMSX reaches a ROM in share/systemroms through the file pool by
+    SHA1 alone, a filename being looked up only beside the machine's XML
+    (src/memory/Rom.cc:121-170). Read as names the core opens, the labels
+    were added to the KNULLI and Batocera exports."""
+
+    def test_no_systemrom_label_reads_as_a_name(self):
+        import yaml
+
+        from validation import opens_by_name
+
+        profile = yaml.safe_load((ROOT / "emulators" / "openmsx.yml").read_text())
+        named = [
+            entry["name"] for entry in profile["files"]
+            if "share/systemroms/" in str(entry.get("path"))
+            and opens_by_name(entry, profile)
+        ]
+        self.assertEqual(named, [])
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_e2e.py

@@ -5541,6 +5541,40 @@ struct BurnDriver BurnDrvneogeo = {
             self.assertEqual(local, path)
             self.assertEqual(status, "name_exact")
 
+    def test_217b_zipped_file_composite_needs_the_member(self):
+        """One member of another name has the zipped_file md5 as composite.
+
+        Batocera looks the member up by name, so the archive is not proof of
+        the entry: a MAME rename leaves exactly that archive behind.
+        """
+        import hashlib as hl
+        import tempfile
+
+        from common import resolve_local_file
+
+        rom = b"inner rom bytes"
+        rom_md5 = hl.md5(rom).hexdigest()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = os.path.join(tmpdir, "foo.zip")
+            with zipfile.ZipFile(path, "w") as zf:
+                zf.writestr("renamed.bin", rom)
+            db = {
+                "files": {"sha1z": {"name": "foo.zip", "path": path, "md5": "f" * 32}},
+                "indexes": {
+                    "by_md5": {}, "by_name": {"foo.zip": ["sha1z"]},
+                    "by_crc32": {}, "by_path_suffix": {},
+                },
+            }
+            entry = {"name": "foo.zip", "zipped_file": "expected.bin", "md5": rom_md5}
+            self.assertNotEqual(
+                resolve_local_file(entry, db)[1], "md5_composite_exact"
+            )
+            with zipfile.ZipFile(path, "w") as zf:
+                zf.writestr("expected.bin", rom)
+            self.assertEqual(
+                resolve_local_file(entry, db), (path, "md5_composite_exact")
+            )
+
     def test_215_check_member_hash_inside_zip(self):
         """zipped_file entries verify the ROM inside the ZIP, not the ZIP."""
         import hashlib as hl

tests/test_foreign_tree_names.py

@@ -93,6 +93,49 @@ class ForeignTreeNames(unittest.TestCase):
         self.assertEqual(status, "data_dir")
         self.assertEqual(Path(path).read_bytes(), b"B" * 64)
 
+    def test_another_casing_in_another_tree_is_not_the_file(self):
+        """reminiscence's music module reunion took skyline's
+        zoneinfo/Indian/Reunion, a time-zone file, on the casefold step."""
+        self._hold("bios/Other/skyline/tzdata/zoneinfo/Indian/Reunion", b"T" * 64)
+        entry = {
+            "name": "reunion", "aliases": ["mod.flashback-reunion"],
+            "category": "game_data", "source_profile": "reminiscence",
+        }
+        self.assertEqual(resolve_local_file(entry, self.db), (None, "not_found"))
+
+
+class RetailDashboardTree(unittest.TestCase):
+    """x1-box asked for C/xbox.xtf: the only copy at that tail sat in hakuX's
+    tree and was set aside, and the name step served Xenia's Xbox 360 font.
+    The retail dashboard both emulators import lives with the console."""
+
+    def test_partition_c_resolves_by_path_for_both_importers(self):
+        import yaml
+
+        from common import load_database
+
+        db_path = REPO_ROOT / "database.json"
+        if not db_path.exists():
+            self.skipTest("database.json absent")
+        db = load_database(str(db_path))
+        for profile in ("x1-box", "hakux"):
+            document = yaml.safe_load(
+                (REPO_ROOT / "emulators" / f"{profile}.yml").read_text()
+            )
+            for entry in document["files"]:
+                dest = str(entry.get("path") or "")
+                if not dest.startswith("C/"):
+                    continue
+                with self.subTest(profile=profile, dest=dest):
+                    asked = {**entry, "source_profile": profile}
+                    cwd = os.getcwd()
+                    os.chdir(REPO_ROOT)
+                    try:
+                        path, status = resolve_local_file(asked, db, dest_hint=dest)
+                    finally:
+                        os.chdir(cwd)
+                    self.assertEqual(status, "path_exact", path)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_profile_sync.py

@@ -333,6 +333,19 @@ class TestGoneUpstreamIsNotAnError(unittest.TestCase):
         self.assertIn("upstream gone", out)
         self.assertIn("upstream error", out)
 
+    def test_triage_names_an_orphaned_pin_and_a_moved_repository(self):
+        """pcem's only problem was an upstream_commit off its dev branch, and
+        the triage line read "1 to review" like any drift."""
+        report = ProfileReport(
+            name="pcem", entries=[], counts={"UNCHECKED": 2},
+            orphaned=["sarah-walker-pcem/pcem: pin 25b54e3f is no ancestor of 4606fb1e"],
+            moved_repos=["a/b answers as c/b (renamed or transferred)"],
+        )
+        self.assertEqual(
+            profile_sync.triage_state(report),
+            "2 to review, 1 orphaned pin, 1 repository moved, 2 unchecked",
+        )
+
 
 class TestWorstStatus(unittest.TestCase):
     def test_gone_beats_everything(self):
@@ -501,6 +514,46 @@ class TestVerifyAtPin(unittest.TestCase):
         self.assertIsNone(profile_sync.cites_another_version(same, "0.16.0"))
         self.assertIsNone(profile_sync.cites_another_version(same, {"libretro": "v0.16.0"}))
 
+    def test_a_profile_with_a_repository_names_a_revision(self):
+        """redream declared libretro/redream, a repository that never
+        existed, with no pin and no date: profile_sync skipped it and no
+        revision answered for the files Batocera, RetroBat and KNULLI ship."""
+        import yaml
+
+        root = Path(__file__).resolve().parents[1] / "emulators"
+        unpinned = [
+            path.stem
+            for path in sorted(root.glob("*.yml"))
+            if (profile := yaml.safe_load(path.read_text(encoding="utf-8")) or {})
+            and (profile.get("source") or profile.get("upstream"))
+            and not any(
+                profile.get(key)
+                for key in ("source_commit", "upstream_commit", "profiled_date")
+            )
+        ]
+        self.assertEqual(unpinned, [])
+
+    def test_a_profile_citing_versions_documents_its_own(self):
+        """ti99sim's core_version went null while doc/CHANGES.html at its pin
+        reads 0.16.0: every (v0.16.0) part then read as another version, and
+        the profile reported 0 to review with nothing checked."""
+        import yaml
+
+        root = Path(__file__).resolve().parents[1] / "emulators"
+        silent = []
+        for path in sorted(root.glob("*.yml")):
+            profile = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            refs = " ".join(
+                str(entry.get("source_ref") or "") for entry in profile.get("files") or []
+            )
+            if not profile_sync._VERSION_NOTE_RE.search(refs):
+                continue
+            declared = profile.get("core_version")
+            values = declared.values() if isinstance(declared, dict) else [declared]
+            if not any(values):
+                silent.append(path.stem)
+        self.assertEqual(silent, [])
+
     def test_a_name_stem_elsewhere_does_not_move_a_ref(self):
         """linapple cites the PRAVETS82 option, and the stem also names an install line."""
         lines = ['option(ENABLE_ROM_CLONE_PRAVETS "" OFF)', "", "", "", "", "",
@@ -3203,6 +3256,35 @@ class TestBumpCommit(unittest.TestCase):
         # Nothing was compared upstream, so its pin is not written.
         self.assertNotIn("uphead", text)
 
+    def test_a_profile_one_platform_ships_waits_for_its_build(self):
+        """EmuELEC alone ships RigelEngine and builds its package.mk
+        PKG_VERSION. A bulk advance moved fheroes2 and supertux past that
+        build, and rigelengine is a commit ahead of it already."""
+        root = Path(__file__).resolve().parents[1]
+        path = Path(self.tmp.name) / "rigelengine.yml"
+        path.write_text(
+            SAMPLE.replace("\nnotes:", "\ntype: standalone\ncores: [rigelengine, RigelEngine]\n\nnotes:", 1),
+            encoding="utf-8",
+        )
+        report = ProfileReport(
+            name="rigelengine", repo="o/n", pin="pin", head="newhead", entries=[],
+            counts={}, skipped=profile_sync.NO_SOURCE_REF,
+        )
+        platforms = str(root / "platforms")
+        self.assertEqual(
+            profile_sync.shipping_platforms("rigelengine", self.tmp.name, platforms),
+            ["emuelec"],
+        )
+        with contextlib.redirect_stderr(io.StringIO()) as err:
+            self.assertFalse(bump_commit(path, report, platforms_dir=platforms))
+        self.assertIn("only emuelec ships it", err.getvalue())
+        path.write_text(
+            path.read_text(encoding="utf-8")
+            + 'pin_follows: "EmuELEC builds the branch tip"\n',
+            encoding="utf-8",
+        )
+        self.assertTrue(bump_commit(path, report, platforms_dir=platforms))
+
     def test_any_other_skip_still_holds_the_pin(self):
         report = ProfileReport(
             name="p", repo="o/n", pin="pin", head="newhead", entries=[], counts={},
@@ -3929,6 +4011,50 @@ class TestCollectCitations(unittest.TestCase):
         self.assertIsNone(resolve("munt ROMInfo.cpp", [tree]))
         self.assertIsNone(resolve("mupen64plus Config.cpp", [tree]))
 
+    def test_project_word_resolves_inside_a_submodule(self):
+        """idtech4a++ and starship cite files of a submodule. The declared
+        tree holds only the gitlink, so the parts stayed external and moved
+        unchecked with every bump."""
+        links = ["Q3E/src/main/jni/xash3d/xash3d-fwgs", "tools/Torch", "libs/zlib"]
+        resolve = profile_sync.resolve_through_gitlinks
+        self.assertEqual(
+            resolve("xash3d-fwgs filesystem/filesystem.c", [links]),
+            "Q3E/src/main/jni/xash3d/xash3d-fwgs/filesystem/filesystem.c",
+        )
+        self.assertEqual(
+            resolve("Torch src/Companion.cpp", [links]), "tools/Torch/src/Companion.cpp"
+        )
+        self.assertIsNone(resolve("munt ROMInfo.cpp", [links]))
+        self.assertIsNone(resolve("lib x.c", [["a/lib1", "b/lib2"]]))
+
+    def test_a_submodule_file_is_read_at_its_pinned_commit(self):
+        port = upstream.make_repo("github.com", "glKarin", "com.n0n3m4.diii4a")
+        files = {
+            (port.slug, "p1", ".gitmodules"): [
+                '[submodule "Q3E/src/main/jni/xash3d/xash3d-fwgs"]',
+                "\tpath = Q3E/src/main/jni/xash3d/xash3d-fwgs",
+                "\turl = ../xash3d-fwgs.git",
+            ],
+            ("glKarin/xash3d-fwgs", "s1", "filesystem/filesystem.c"): ["getenv(x)"],
+        }
+
+        def fetch(repo, sha, path, *_):
+            return files.get((repo.slug, sha, path))
+
+        with mock.patch.object(upstream, "fetch_file", side_effect=fetch), \
+                mock.patch.object(
+                    upstream, "list_gitlinks",
+                    return_value={"Q3E/src/main/jni/xash3d/xash3d-fwgs": "s1"},
+                ):
+            target = upstream.through_gitlink(
+                port, "p1",
+                "Q3E/src/main/jni/xash3d/xash3d-fwgs/filesystem/filesystem.c", "c",
+            )
+        self.assertEqual(
+            (target[0].slug, target[1], target[2]),
+            ("glKarin/xash3d-fwgs", "s1", "filesystem/filesystem.c"),
+        )
+
     def test_run_is_rendered_with_its_written_path(self):
         parts = [RefPart("mupen64plus-video-rice/upstream/src/Config.cpp", 38, 38, "38")]
         self.assertEqual(
@@ -5799,6 +5925,24 @@ class TestBranchProfiles(unittest.TestCase):
             ["draft", "edited", "new"],
         )
 
+    def test_a_fork_clone_compares_with_the_project(self):
+        """origin is the fork, pushed: origin/main is HEAD, and the default
+        base found nothing to check while the PR's CI failed."""
+        base = subprocess.run(
+            ["git", "rev-parse", "main"], cwd=self.root, capture_output=True,
+            text=True, check=True,
+        ).stdout.strip()
+        self._git("remote", "add", "origin", "https://github.com/someone/retrobios")
+        self._git("remote", "add", "upstream", "https://github.com/Abdess/retrobios")
+        self._git("update-ref", "refs/remotes/origin/main", "HEAD")
+        self._git("update-ref", "refs/remotes/upstream/main", base)
+        args = profile_sync.build_parser().parse_args(["--changed"])
+        profiles = {"kept": {}, "edited": {}, "new": {}, "draft": {}}
+        self.assertEqual(
+            list(profile_sync.select_profiles(profiles, args)),
+            ["draft", "edited", "new"],
+        )
+
 
 class TestBranchCheckIsWired(unittest.TestCase):
     """CONTRIBUTING and CI name the branch's profiles through the flag."""

tests/test_region.py

@@ -509,6 +509,11 @@ class OneCodeOneRegionAnswer(unittest.TestCase):
         profiles = common.load_emulator_profiles(self.PROFILES)
         bios_sms = next(e for e in profiles["clk"]["files"] if e["name"] == "bios.sms")
         self.assertIn("oceania", bios_sms["region"])
+        # A Mexican export cartridge is analysed as Europe or USA and boots the
+        # western BIOS: only Japan selects the other one.
+        self.assertEqual(
+            region.rank(region.normalize_declared(bios_sms["region"]), ["mexico"]), 0
+        )
         tagged = {
             e["name"]: e.get("region") for e in profiles["kronos"]["files"]
             if e.get("archive") == "stvbios.zip" and e["name"].startswith("epr-")
@@ -1379,5 +1384,32 @@ class YmirLibretroRegions(unittest.TestCase):
                              if e["name"] == "sega_101.bin"))
 
 
+class MachineModelRomsCarryNoTerritory(unittest.TestCase):
+    """XRoar reads d200e_1.rom for the dragon200e machine and alice.rom for
+    the alice machine (src/xroar.c:437-444, 552-558), and VICE the Japanese
+    kernal and chargen for the C64 JAP and VIC-1001 models, which a file name
+    tagged (Japan) never selects (libretro-core.c:2018-2034 sets NTSC). The
+    site told a Spanish Dragon 64 owner to start with d200e_1.rom."""
+
+    CASES = {
+        "xroar": ("d200e_1.rom", "d200e_2.rom", "d200e_26.rom", "alice.rom"),
+        "x64sdl": ("jpkernal", "jpchrgen"),
+        "vice_xvic": ("kernal.901486-02.bin", "chargen-901460-02.bin"),
+    }
+
+    def test_model_roms_are_unregioned(self):
+        profiles = common.load_emulator_profiles(
+            os.path.join(os.path.dirname(__file__), "..", "emulators")
+        )
+        for profile, names in self.CASES.items():
+            for entry in profiles[profile]["files"]:
+                if entry["name"] in names:
+                    with self.subTest(profile=profile, name=entry["name"]):
+                        self.assertNotIn("region", entry)
+        self.assertFalse(
+            [e["name"] for e in profiles["c64-emu"]["files"] if e.get("region")]
+        )
+
+
 if __name__ == "__main__":
     unittest.main()