From 36427edbb2dd2d82b354c53d99bdecd72270cdf5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:11:48 +0200 Subject: [PATCH] fix: point source refs at paths held at the pin trident cites the Panda3DS fork 3dsTrident vendors as a submodule, now its upstream at the submodule pin 69095ee6. pce-emu, uqm and stonesoup wrote paths from a subdirectory (PCE.emu/, sc2/, crawl-ref/), nesoid named two Java files without their directory; lines unchanged, each read at the pin. The pages no longer take a binary citation (DesktopKinect.dll, an address continuing a BAM.dll range) for a source path, and a Forgejo tree is read page by page: mirrors/Ryujinx holds 4704 entries, 1000 a page. --- emulators/nesoid.yml | 2 +- emulators/pce-emu.yml | 4 +- emulators/stonesoup.yml | 6 +- emulators/trident.yml | 9 ++- emulators/uqm.yml | 14 ++-- scripts/profile_sync.py | 27 +------ scripts/sitesources.py | 9 ++- scripts/sourceref.py | 26 +++++++ scripts/upstream.py | 20 ++++- tests/test_source_ref_targets.py | 125 +++++++++++++++++++++++++++++++ 10 files changed, 198 insertions(+), 44 deletions(-) create mode 100644 tests/test_source_ref_targets.py diff --git a/emulators/nesoid.yml b/emulators/nesoid.yml index 2a64be26..e9821479 100644 --- a/emulators/nesoid.yml +++ b/emulators/nesoid.yml @@ -141,7 +141,7 @@ files: 16 KB further in, so such a file needs at least 16656 bytes. Anything else is read as a raw 4352 byte image. A failed open or a short read leaves the Game Genie off and the content still runs. No hash is compared. - source_ref: "app/jni/neslib/cart.c:475-521 (open and format branch), app/jni/neslib/svga.c:132-140 (setter), app/jni/neslib/drivers/android/nesengine.cpp:242-243, app/jni/neslib/fce.c:1201-1203 (call site), EmulatorActivity.java:870-874, EmulatorSettingsFragment.java:179-193 (picker)" + source_ref: "app/jni/neslib/cart.c:475-521 (open and format branch), app/jni/neslib/svga.c:132-140 (setter), app/jni/neslib/drivers/android/nesengine.cpp:242-243, app/jni/neslib/fce.c:1201-1203 (call site), app/src/main/java/com/androidemu/nes/EmulatorActivity.java:870-874, app/src/main/java/com/androidemu/nes/EmulatorSettingsFragment.java:179-193 (picker)" - name: ".pal" system: nintendo-nes diff --git a/emulators/pce-emu.yml b/emulators/pce-emu.yml index 82f2af7a..fd27e46e 100644 --- a/emulators/pce-emu.yml +++ b/emulators/pce-emu.yml @@ -52,7 +52,7 @@ files: validation: [existence, size] min_size: 262144 category: bios - source_ref: "PCE.emu/src/main/Main.cc:80-91 (throws without it), PCE.emu/src/main/options.cc:275-301 (path resolution), PCE.emu/src/main/EmuMenuViews.cc:176-191 (selection), src/pce_fast/pce.cpp:477-483 + src/pce_fast/huc.cpp:284-304 (default module load), src/pce/pce.cpp:645-653 (accurate module load), src/pce_fast/pce.cpp:663 + src/pce/pce.cpp:1094 (name)" + source_ref: "PCE.emu/src/main/Main.cc:80-91 (throws without it), PCE.emu/src/main/options.cc:275-301 (path resolution), PCE.emu/src/main/EmuMenuViews.cc:176-191 (selection), PCE.emu/src/pce_fast/pce.cpp:477-483, PCE.emu/src/pce_fast/huc.cpp:284-304 (default module load), PCE.emu/src/pce/pce.cpp:645-653 (accurate module load), PCE.emu/src/pce_fast/pce.cpp:663, PCE.emu/src/pce/pce.cpp:1094 (name)" note: > min_size comes from the fixed 262144-byte read in pce_fast (huc.cpp:302) with Stream::read throwing on a short file @@ -64,7 +64,7 @@ files: required: false validation: [existence] category: bios - source_ref: "src/pce/pce.cpp:512-551 (DetectGECD), src/pce/pce.cpp:644-646 (gecdbios branch), src/pce/huc.cpp:225-316 (size-agnostic read), src/pce/pce.cpp:1095 (name)" + source_ref: "PCE.emu/src/pce/pce.cpp:512-551 (DetectGECD), PCE.emu/src/pce/pce.cpp:644-646 (gecdbios branch), PCE.emu/src/pce/huc.cpp:225-316 (size-agnostic read), PCE.emu/src/pce/pce.cpp:1095 (name)" note: > Reached only when the accurate module runs and DetectGECD matches the disc, since pce_fast has no Games Express branch. Served through the same diff --git a/emulators/stonesoup.yml b/emulators/stonesoup.yml index 8fdc9a6c..c2d1339e 100644 --- a/emulators/stonesoup.yml +++ b/emulators/stonesoup.yml @@ -16,10 +16,10 @@ notes: | Accepts .crawlrc config files as content (supported_extensions: crawlrc). The libretro wrapper sets -dir to the content directory. - ref: libretro/libretro.cc:178-198 (main_wrap() passes -dir content_dir) + ref: crawl-ref/libretro/libretro.cc:178-198 (main_wrap() passes -dir content_dir) Game data loaded at runtime from dat/ tree via datafile_path(). - ref: source/files.cc:375-458 (_get_base_dirs() + datafile_path()) + ref: crawl-ref/source/files.cc:375-458 (_get_base_dirs() + datafile_path()) No DATA_DIR_PATH defined in Makefile.libretro, so it falls back to SysEnv.crawl_dir. Data is NOT compiled into the binary. @@ -37,7 +37,7 @@ notes: | data_directories: - ref: stonesoup installed_on: [recalbox] - source_ref: "source/files.cc:375-458" + source_ref: "crawl-ref/source/files.cc:375-458" files: [] # no BIOS/firmware, game data loaded via data_directories diff --git a/emulators/trident.yml b/emulators/trident.yml index 0ffd332b..aab60c74 100644 --- a/emulators/trident.yml +++ b/emulators/trident.yml @@ -3,9 +3,10 @@ display_name: "Nintendo - 3DS (Trident)" type: libretro core_classification: community_fork source: https://github.com/DanAlexMorton/3dsTrident -upstream: https://github.com/wheremyfoodat/Panda3DS +upstream: https://github.com/DanAlexMorton/Panda3DS profiled_date: "2026-03-25" source_commit: "b97b0befa326105e181838bc6fa11c546de8294d" +upstream_commit: "69095ee6d68d21eacac8d1517f0b7b4ed27884d3" core_version: git library_name: "Trident" @@ -24,8 +25,10 @@ notes: >- Files are resolved from the RetroArch save directory (/Emulator Files/sysdata/), not the system directory, or from the working directory when the frontend gives none. The core reports - library_name "Trident". The wrapper is vendor/Panda3DS at 69095ee6 - (DanAlexMorton/Panda3DS src/libretro_core.cpp:30, 243, 280-285). + library_name "Trident" (src/libretro_core.cpp:29, 243, 280-285). + 3dsTrident builds the vendor/Panda3DS submodule, DanAlexMorton/Panda3DS + at 69095ee6, a fork of wheremyfoodat/Panda3DS from their merge base + 944b9892; the libretro wrapper and the files cited below are in that tree. files: - name: aes_keys.txt diff --git a/emulators/uqm.yml b/emulators/uqm.yml index bbbf4f84..54243aef 100644 --- a/emulators/uqm.yml +++ b/emulators/uqm.yml @@ -60,7 +60,7 @@ files: validation: [md5] description: "UQM core game data archive" note: "Published apart from the source release. The RetroPie scriptmodule downloads it into content/packages." - source_ref: "build/win32_install/packages.nsh:3-5; sc2/build/win32_install/uqm-installer.nsi:289-306" + source_ref: "sc2/build/win32_install/packages.nsh:3-5; sc2/build/win32_install/uqm-installer.nsi:289-306" - name: uqm-0.8.0-3domusic.uqm path: uqm/content/addons/uqm-0.8.0-3domusic.uqm @@ -70,7 +70,7 @@ files: md5: "9e5801d45ca12028b486cdeb83568c02" validation: [md5] description: "UQM 3DO music track pack" - source_ref: "build/win32_install/packages.nsh:6-8; sc2/build/win32_install/uqm-installer.nsi:315-327, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:6-8; sc2/build/win32_install/uqm-installer.nsi:315-327, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" - name: uqm-0.8.0-voice.uqm path: uqm/content/addons/uqm-0.8.0-voice.uqm @@ -80,7 +80,7 @@ files: md5: "fc89f77d7b66c2669abca6c157f5259a" validation: [md5] description: "UQM alien-race voice-over pack" - source_ref: "build/win32_install/packages.nsh:9-11; sc2/build/win32_install/uqm-installer.nsi:329-341, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:9-11; sc2/build/win32_install/uqm-installer.nsi:329-341, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" - name: uqm-remix-disc1.uqm path: uqm/content/addons/uqm-remix-disc1.uqm @@ -90,7 +90,7 @@ files: md5: "09f242d8d72166d1d5ccbd3d99c93e7d" validation: [md5] description: "Precursor Remix disc 1 (modern remixes)" - source_ref: "build/win32_install/packages.nsh:12-14; sc2/build/win32_install/uqm-installer.nsi:345-358, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:12-14; sc2/build/win32_install/uqm-installer.nsi:345-358, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" - name: uqm-remix-disc2.uqm path: uqm/content/addons/uqm-remix-disc2.uqm @@ -100,7 +100,7 @@ files: md5: "fbc8bdcb709939d559d8c7216ad15cc2" validation: [md5] description: "Precursor Remix disc 2" - source_ref: "build/win32_install/packages.nsh:15-17, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:15-17, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" - name: uqm-remix-disc3.uqm path: uqm/content/addons/uqm-remix-disc3.uqm @@ -110,7 +110,7 @@ files: md5: "5ccc6d4ac301ae98e172ac6835dcdead" validation: [md5] description: "Precursor Remix disc 3" - source_ref: "build/win32_install/packages.nsh:18-20, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:18-20, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" - name: uqm-remix-disc4-1.uqm path: uqm/content/addons/uqm-remix-disc4-1.uqm @@ -120,4 +120,4 @@ files: md5: "3fb63f4ac514343ed4b4b5b194c413fb" validation: [md5] description: "Precursor Remix disc 4-1" - source_ref: "build/win32_install/packages.nsh:21-23, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" + source_ref: "sc2/build/win32_install/packages.nsh:21-23, sc2/src/options.c:393-402, 518-551, sc2/src/uqm/setup.c:118-132" diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 5e390a67..a3b0010d 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -46,6 +46,8 @@ from sourceref import ( # noqa: F401 (re-exported for callers and tests) # no _PROSE_LIST_DELIMITERS, _REF_RE, _SPLIT_RE, + _BINARY_SUFFIXES, + _HEX_ADDRESS_RE, PROSE_CITE_RE, PROSE_CONT_RE, PROSE_CONT_SPACED_RE, @@ -58,6 +60,7 @@ from sourceref import ( # noqa: F401 (re-exported for callers and tests) # no _prose_external, _prose_runs, _trim_prose, + is_binary_citation, parse_source_ref, ref_prose_text, ref_role, @@ -195,33 +198,9 @@ def is_external_citation(path: str) -> bool: return "/" not in head and "." not in head -_BINARY_SUFFIXES = ( - ".dll", ".exe", ".so", ".dylib", ".apk", ".elf", ".o", ".a", ".jar", ".dex", -) -_HEX_ADDRESS_RE = re.compile(r"\b0x[0-9a-fA-F]{2,}") _VERSION_WORD_RE = re.compile(r"v?\d+(\.\d+)+[\w.-]*") -def is_binary_citation(path: str) -> bool: - """True when a ref locates code inside a shipped binary, not a source file. - - `BAM.dll 1.5-408 .text:0x100c3960-0x100c3a97`, `libmain.so LoadNES - 0x3bb49` and `.rdata dialog filter` were read by disassembly. No revision - of any repository holds those bytes, so the ref can neither anchor nor - break: 77 parts of `future-pinball-fploader` alone were reported GONE. - """ - if _HEX_ADDRESS_RE.search(path): - return True - if " " not in path: - # A bare `DesktopKinect.dll` names a shipped library, never a source - # file: no directory, no line, only its import table to read. - return "/" not in path and path.lower().endswith(_BINARY_SUFFIXES) - head = path.split(" ", 1)[0] - if head.lower().endswith(_BINARY_SUFFIXES): - return True - return head.startswith(".") and "/" not in head - - def strip_repo_word( path: str, known: frozenset[str] | set[str] = frozenset() ) -> str: diff --git a/scripts/sitesources.py b/scripts/sitesources.py index adac7dc2..85cee28c 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -14,7 +14,9 @@ import urllib.request import upstream from siterender import _by_mode -from sourceref import _prose_runs, ref_role, source_ref_values, split_source_ref +from sourceref import ( + _prose_runs, is_binary_citation, ref_role, source_ref_values, split_source_ref, +) def _repo_pins(profile: dict, field: str, label: str) -> list[tuple[str, str, str]]: @@ -309,7 +311,10 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str: # broke 470 links. Two repositories and no tree to read: no link. selected: tuple[upstream.Repo, str] | None = None real_path = part.path - if path_re.fullmatch(part.path): + # A shipped binary (`DesktopKinect.dll`, `0x10037130-0x100371de` + # continuing a BAM.dll range) is no path of either repository: + # without a cached tree the single declared one took it. + if path_re.fullmatch(part.path) and not is_binary_citation(part.path): if current_role: # Only the repository the part names, or no link at all. selected = by_role.get(current_role) diff --git a/scripts/sourceref.py b/scripts/sourceref.py index 9ec48688..33589874 100644 --- a/scripts/sourceref.py +++ b/scripts/sourceref.py @@ -25,6 +25,32 @@ _LOCATED_RE = re.compile(r"^[^\s:]+:\d+(?:-\d+)?$") REPO_ROLES = ("source", "upstream") +_BINARY_SUFFIXES = ( + ".dll", ".exe", ".so", ".dylib", ".apk", ".elf", ".o", ".a", ".jar", ".dex", +) +_HEX_ADDRESS_RE = re.compile(r"\b0x[0-9a-fA-F]{2,}") + + +def is_binary_citation(path: str) -> bool: + """True when a ref locates code inside a shipped binary, not a source file. + + `BAM.dll 1.5-408 .text:0x100c3960-0x100c3a97`, `libmain.so LoadNES + 0x3bb49` and `.rdata dialog filter` were read by disassembly. No revision + of any repository holds those bytes, so the ref can neither anchor nor + break: 77 parts of `future-pinball-fploader` alone were reported GONE. + """ + if _HEX_ADDRESS_RE.search(path): + return True + if " " not in path: + # A bare `DesktopKinect.dll` names a shipped library, never a source + # file: no directory, no line, only its import table to read. + return "/" not in path and path.lower().endswith(_BINARY_SUFFIXES) + head = path.split(" ", 1)[0] + if head.lower().endswith(_BINARY_SUFFIXES): + return True + return head.startswith(".") and "/" not in head + + def ref_role(path: str) -> tuple[str, str]: """The role word a cited path opens with, and the path without it.""" word, _, rest = path.partition(" ") diff --git a/scripts/upstream.py b/scripts/upstream.py index 69fdae40..56391ce8 100644 --- a/scripts/upstream.py +++ b/scripts/upstream.py @@ -780,12 +780,28 @@ def list_tree( payload = _api(url, cache_dir, offline) if not isinstance(payload, dict): return [], True + entries = list(payload.get("tree") or []) + truncated = bool(payload.get("truncated")) + # Forgejo pages a recursive tree, 1000 entries at a time, and says how + # many there are: mirrors/Ryujinx holds 4704, and read as one page every + # citation past the first thousand was unverifiable. GitHub reports no + # count, its truncation is final. + total = payload.get("total_count") + if truncated and repo.family == "forgejo" and isinstance(total, int): + for page in range(2, MAX_TREE_PAGES + 1): + if len(entries) >= total: + break + more = _api(f"{url}&page={page}", cache_dir, offline) + if not isinstance(more, dict) or not more.get("tree"): + break + entries.extend(more["tree"]) + truncated = len(entries) < total paths = [ entry["path"] - for entry in payload.get("tree") or [] + for entry in entries if isinstance(entry, dict) and entry.get("type") == "blob" and entry.get("path") ] - return paths, bool(payload.get("truncated")) + return paths, truncated def list_gitlinks( diff --git a/tests/test_source_ref_targets.py b/tests/test_source_ref_targets.py new file mode 100644 index 00000000..53e4f973 --- /dev/null +++ b/tests/test_source_ref_targets.py @@ -0,0 +1,125 @@ +"""Each cited path is one a declared repository holds at its pin. + +Seven profiles cited paths their repositories do not hold. trident wraps +the Panda3DS fork vendored as a submodule, so 3dsTrident holds none of the +emulator sources it cites. pce-emu, uqm and stonesoup wrote their paths from +a subdirectory of the repository (PCE.emu/, sc2/, crawl-ref/), and nesoid +named two Java files without their directory. future-pinball-fploader and +nesoid continue a binary citation with a bare address or library name, +which the pages took for a path of the source repository. The Ryujinx +mirror pages its tree, and read as one page it could not confirm a path. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import sitesources # noqa: E402 +import upstream # noqa: E402 +from common import load_emulator_profiles # noqa: E402 +from sourceref import ref_role, source_ref_values, split_source_ref # noqa: E402 + +PROFILES = ("trident", "future-pinball-fploader", "pce-emu", "uqm", "nesoid", + "ryujinx", "stonesoup") + + +class BinaryContinuationsAreNotLinked(unittest.TestCase): + PROFILE = { + "emulator": "future-pinball-fploader", "type": "standalone", + "source": "https://github.com/ravarcade/BAM_FPloader", + "source_commit": "e" * 40, + } + + def tearDown(self): + sitesources._tree_paths.cache_clear() + + def _render(self, value: str) -> str: + original = sitesources._tree_paths + sitesources._tree_paths = lambda repo, pin: None # type: ignore[assignment] + try: + return sitesources._source_ref_markdown(self.PROFILE, value) + finally: + sitesources._tree_paths = original + + def test_an_address_continuing_a_binary_range(self): + text = self._render("BAM.dll 1.5-408 .text:0x100ae843-0x100ae8a7, 0x10037130-0x100371de") + self.assertNotIn("](https://", text) + self.assertIn("`0x10037130-0x100371de`", text) + + def test_a_bare_library_name(self): + text = self._render("DesktopKinect.dll, Kinect.dll import directories") + self.assertNotIn("](https://", text) + + def test_a_source_path_is_still_linked(self): + text = self._render("FPLoader.cpp:259-305") + self.assertIn("](https://github.com/ravarcade/BAM_FPloader/blob/", text) + + +class ForgejoTreesArePaged(unittest.TestCase): + """git.ngram.ca returns 1000 entries a page and the total.""" + + def test_every_page_is_read(self): + repo = upstream.parse_repo("https://git.ngram.ca/mirrors/Ryujinx") + pages = { + 1: [{"type": "blob", "path": f"a/{i}.cs"} for i in range(1000)], + 2: [{"type": "blob", "path": "src/Ryujinx.HLE/FileSystem/VirtualFileSystem.cs"}, + {"type": "tree", "path": "src"}], + } + + def fake_api(url, cache_dir, offline, *args, **kwargs): + page = int(url.rsplit("&page=", 1)[1]) if "&page=" in url else 1 + return {"tree": pages.get(page, []), "truncated": True, "total_count": 1002} + + with mock.patch.object(upstream, "_api", side_effect=fake_api): + paths, truncated = upstream.list_tree(repo, "7" * 40, "unused") + self.assertFalse(truncated) + self.assertEqual(len(paths), 1001) + self.assertIn("src/Ryujinx.HLE/FileSystem/VirtualFileSystem.cs", paths) + + def test_a_github_truncation_stays_final(self): + repo = upstream.parse_repo("https://github.com/o/r") + payload = {"tree": [{"type": "blob", "path": "x"}], "truncated": True} + with mock.patch.object(upstream, "_api", return_value=payload) as api: + paths, truncated = upstream.list_tree(repo, "7" * 40, "unused") + self.assertTrue(truncated) + self.assertEqual(api.call_count, 1) + + +class TheSevenProfilesCiteHeldPaths(unittest.TestCase): + """Read against the cached tree listings, skipped where none is cached.""" + + def test_every_cited_path_is_held(self): + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False) + missing, checked = [], 0 + for key in PROFILES: + profile = profiles[key] + candidates = sitesources._forge_sources(profile) + values = [e["source_ref"] for e in profile.get("files") or [] + if isinstance(e, dict) and e.get("source_ref")] + values += [d["source_ref"] for d in profile.get("data_directories") or [] + if isinstance(d, dict) and d.get("source_ref")] + for value in values: + for _label, refs in source_ref_values(value): + for part in split_source_ref(refs): + _role, path = ref_role(part.path) + if " " in path or sitesources.is_binary_citation(path): + continue + verdicts = [sitesources.tree_holds(r, p, path) for r, p in candidates] + if None in verdicts: + continue + checked += 1 + if not any(verdicts): + missing.append((key, path)) + if not checked: + raise unittest.SkipTest("no cached tree listings") + self.assertEqual(missing, []) + + +if __name__ == "__main__": + unittest.main()