From 8dcf53162500cb4efd9c45bcb003942d12d24150 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:57:01 +0100 Subject: [PATCH] fix: report pinned trees the source record lacks --- scripts/check_freshness.py | 18 ++++++++++----- scripts/sitesources.py | 13 ++++++----- tests/test_profile_sync.py | 3 ++- tests/test_site_r4_merge.py | 32 ++++++++++++++++++++++++++ website/src/data/source-paths.json | 36 ++++++++++++++++++++++++++++++ wiki/tools.md | 3 ++- 6 files changed, 91 insertions(+), 14 deletions(-) diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index a1116142d..e53c286ca 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -1242,8 +1242,10 @@ def _check_installer_pins() -> Finding: def check_source_paths(profiles: dict) -> Finding: """The site links a cited path only where source-paths.json records it - in the tree the profile pins. A recorded tree no profile pins any more - means a pin moved and `sitesources.py --fetch` was not run after it.""" + in the tree the profile pins. A recorded tree no profile pins any more, + or a pinned tree the record lacks, means a pin moved or a profile was + added and `sitesources.py --fetch` was not run after it. Every tree a run + tries is recorded, read or not, so a lacking one was never tried.""" import sitesources # noqa: PLC0415 pinned = { @@ -1256,14 +1258,18 @@ def check_source_paths(profiles: dict) -> Finding: except (OSError, ValueError) as exc: return Finding("profiles", "source-paths.json", ERROR, detail=str(exc)) orphaned = sorted(recorded - pinned) + unrecorded = sorted(pinned - recorded) + problems = [f"{len(found)} {words}" for found, words in ( + (orphaned, "recorded trees no profile pins"), + (unrecorded, "pinned trees not recorded"), + ) if found] return Finding( "profiles", "source-paths.json", - STALE if orphaned else OK, + STALE if problems else OK, local=f"{len(recorded)} trees", - detail=(f"{len(orphaned)} recorded trees no profile pins: run " - "scripts/sitesources.py --fetch") if orphaned - else "every recorded tree is a pinned one", + detail=(", ".join(problems) + ": run scripts/sitesources.py --fetch") if problems + else "every pinned tree is recorded, and only those", ) diff --git a/scripts/sitesources.py b/scripts/sitesources.py index 55dc2f5e9..e631a9774 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -311,9 +311,9 @@ def build_source_paths( if tree is None: unread.add(tree_key(repo, pin)) continue - found = paths & tree - if found: - held.setdefault(tree_key(repo, pin), set()).update(found) + # Recorded even when it holds no cited path, so a pinned tree + # missing from the record is one no run has read. + held.setdefault(tree_key(repo, pin), set()).update(paths & tree) return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread) @@ -323,12 +323,13 @@ def keep_unread( """Carry over what the last record held for the trees this run could not read. A tree key names a pinned sha, so what it held then it holds now. A run - that GitLab rate-limited dropped about twenty trees it never read. + that GitLab rate-limited dropped about twenty trees it never read. A tree + no run could read is recorded empty: confirming no path, it links none, + and check_freshness tells it from a pin no run has tried. """ kept = dict(held) for key in unread: - if key not in kept and key in previous: - kept[key] = previous[key] + kept.setdefault(key, previous.get(key, [])) return dict(sorted(kept.items())) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index ce62e6cc8..54eabaed1 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4882,7 +4882,8 @@ class AnUnreadTreeKeepsItsRecord(unittest.TestCase): kept = sitesources.keep_unread( {"a@1": ["x.c"]}, ["b@2", "c@3"], {"b@2": ["y.c"], "a@1": ["old.c"]} ) - self.assertEqual(kept, {"a@1": ["x.c"], "b@2": ["y.c"]}) + # c@3 was never read: recorded empty, it confirms no path. + self.assertEqual(kept, {"a@1": ["x.c"], "b@2": ["y.c"], "c@3": []}) class FreshRefusesOffline(unittest.TestCase): diff --git a/tests/test_site_r4_merge.py b/tests/test_site_r4_merge.py index 6b7d51031..c1d582e5a 100644 --- a/tests/test_site_r4_merge.py +++ b/tests/test_site_r4_merge.py @@ -13,6 +13,7 @@ import glob import json import re import sys +import tempfile import unittest from pathlib import Path from unittest import mock @@ -123,6 +124,37 @@ class SourcePathsFreshness(unittest.TestCase): self.assertEqual(finding.status, check_freshness.STALE) self.assertIn("sitesources.py --fetch", finding.detail) + def test_a_pinned_tree_never_read_is_stale(self): + """fbneo moved to two new pins while fbneo_cps12 kept the old ones: + no recorded tree was orphaned, so the check said OK and 174 citations + stayed unlinked.""" + import sitesources # noqa: PLC0415 + + profile = _profile("geolith") + trees = [sitesources.tree_key(r, p) for r, p in sitesources.profile_trees(profile)] + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + for key in trees: + record.pop(key, None) + # Kept pinned by another profile, as fbneo_cps12 kept the old FBNeo trees. + record.update({key: [] for key in trees[1:]}) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source-paths.json" + path.write_text(json.dumps(record), encoding="utf-8") + with mock.patch.object(sitesources, "SOURCE_PATHS", path): + finding = check_freshness.check_source_paths({"geolith": profile}) + self.assertEqual(finding.status, check_freshness.STALE) + self.assertIn("1 pinned trees not recorded", finding.detail) + + def test_a_tree_read_with_no_cited_path_is_recorded(self): + import sitesources # noqa: PLC0415 + + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40, + "files": [{"source_ref": "src/a.c:5"}]} + with mock.patch.object(sitesources, "_cached_tree", return_value=frozenset({"x.c"})): + held, unread = sitesources.build_source_paths({"p": profile}, [Path("tmp")]) + self.assertEqual(held, {"github.com/o/r@" + "e" * 40: []}) + self.assertEqual(unread, []) + class OptionsGlanceFromTheParser(unittest.TestCase): """--declared-only and --offline reached install.py and its Options diff --git a/website/src/data/source-paths.json b/website/src/data/source-paths.json index c7e9e7996..52daf59bf 100644 --- a/website/src/data/source-paths.json +++ b/website/src/data/source-paths.json @@ -1,7 +1,10 @@ { + "archive.softwareheritage.org/softwareheritage/revision@dc20a615275f66bee20a4fd851ef0231daca4f14": [], + "bitbucket.org/linuxwolf6/splitwolf@f6c00044cdef00a31c66307d8b7cb1fb4ddf8c67": [], "codeberg.org/iyzsong/eternal-libretro@13adf143aea822ebc1ee23722a80e93a7979e4c9": [ "libretro.c" ], + "codeberg.org/iyzsong/gam4980@32c531d3e7159081afd8dee72e8266153821095d": [], "codeberg.org/schelling/dosbox-pure-unleashed@4a11412248ca4c862751a7d9e6818023795031e9": [ "Makefile", "main.cpp" @@ -409,6 +412,7 @@ "plastic_tui/src/ui.rs", "plastic_ui/src/main.rs" ], + "github.com/AppleWin/AppleWin@1f3cadb88d1158c139d2e3a7b864d91836856948": [], "github.com/AppleWin/AppleWin@3e8054b4627624398e4589f7f27b3d40a6b9718e": [ "resource/resource.h", "source/CardManager.cpp", @@ -516,6 +520,7 @@ "source/bus-main-m68k.c", "source/bus-sub-m68k.c" ], + "github.com/Clownacy/clownmdemu-libretro@d43c2708b0a31c285ce16724b6c4a2e92af07346": [], "github.com/ColinPitrat/caprice32@8c778bf072a06fb7c23b9cccd179041db7f79a52": [ "cap32.cfg", "rom/MF2.rom", @@ -597,6 +602,7 @@ "Source/SysPosix/CMakeLists.txt", "Source/Utility/Paths.cpp" ], + "github.com/DanAlexMorton/3dsTrident@b97b0befa326105e181838bc6fa11c546de8294d": [], "github.com/DanAlexMorton/Panda3DS@69095ee6d68d21eacac8d1517f0b7b4ed27884d3": [ "src/core/crypto/aes_engine.cpp", "src/core/loader/ncch.cpp", @@ -855,6 +861,7 @@ "Hurrican/src/Texts.cpp", "Hurrican/src/Tileengine.cpp" ], + "github.com/Hydr8gon/NooDS@4cb1439c330248b26cdf5f8dbcfa4e5ac53dcda0": [], "github.com/JACoders/OpenJK@260c59c2907187af555a676fe0cc798893bf7757": [ "code/CMakeLists.txt", "code/qcommon/files.cpp", @@ -1138,6 +1145,7 @@ "pcsx2/VMManager.cpp", "pcsx2/ps2/BiosTools.cpp" ], + "github.com/PCSX2/pcsx2@c10a5c9ad951c517dc48b722b5b84eb9bf7fdb42": [], "github.com/PCSX2/pcsx2@f0523f8acb1d3a765ee73b74cffec63093569199": [ "common/FileSystem.cpp", "pcsx2/CDVD/CDVD.cpp", @@ -1198,6 +1206,7 @@ "Source/Core/DiscIO/NANDImporter.cpp", "Source/Core/DolphinQt/MenuBar.cpp" ], + "github.com/RPCS3/rpcs3@1f080485b95db2700c5cf244b1314f73b6cf564e": [], "github.com/RPCS3/rpcs3@c365ea76a552961ff078ea1a46da9d4b07729674": [ "rpcs3/Crypto/key_vault.h", "rpcs3/Emu/Cell/Modules/cellFont.cpp", @@ -1226,6 +1235,7 @@ "RSDKv5/RSDK/Core/Reader.cpp", "RSDKv5/RSDK/User/Core/UserCore.cpp" ], + "github.com/RSDKModding/Sonic-Mania-Decompilation@9dc699428420d752af9767bdb13f585ee0881bc0": [], "github.com/Rakashazi/emu-ex-plus-alpha@1c12fac5ce49badaadff2e2f210dcc30b89f4943": [ "2600.emu/CMakeLists.txt", "2600.emu/metadata/conf.mk", @@ -1730,6 +1740,7 @@ "desmume/src/NDSSystem.cpp", "desmume/src/NDSSystem.h" ], + "github.com/TASEmulators/fceux@d339f1fa0cd8b69cc637a0de3fb3f04cb28748ea": [], "github.com/TASEmulators/freej2me-plus@3933e75ef927a95eab9f5e065f0b76dabf336069": [ "build.xml", "src/javax/microedition/media/Manager.java", @@ -1864,6 +1875,7 @@ "wadsrc_lights/CMakeLists.txt", "wadsrc_widepix/CMakeLists.txt" ], + "github.com/VICE-Team/svn-mirror@46b1bc4393251f838ce3dadd241c1f567d9371d1": [], "github.com/VICE-Team/svn-mirror@81546e60c222fe4aef750b890b0dbe3a5ac280d8": [ "vice/configure.ac", "vice/data/C64/Makefile.am", @@ -2472,6 +2484,7 @@ "mia/system/neo-geo-aes.cpp", "mia/system/neo-geo-mvs.cpp" ], + "github.com/atari800/atari800@98cc2dc749867c9c7317a0271ba062a6c797f886": [], "github.com/audetto/AppleWin@0a89db3d593a74d04590b902fb7206723591f445": [ "source/frontends/common2/gnuframe.cpp" ], @@ -2770,6 +2783,7 @@ "src/sdl/main.cpp", "src/sdl/ti994a-sdl.cpp" ], + "github.com/clydeshaffer/GameTankEmulator@be32deab74b7778647273fa5616d1ea92e3e1267": [], "github.com/coelckers/prboom-plus@969515162c5aebea4ad7a125ee178dcad3d576ad": [ "prboom2/CMakeLists.txt", "prboom2/data/CMakeLists.txt", @@ -3167,6 +3181,7 @@ "core/loadrom.c", "libretro/libretro.c" ], + "github.com/emuplace/sudachi.emuplace.app@c7431bd420d66c54df277ba0a0b71de42cf0a5bf": [], "github.com/etlegacy/etlegacy@add16a92f4b10c2955136e3ab7868bfc9bff8fad": [ "README.md", "cmake/ETLBuildMod.cmake", @@ -3182,6 +3197,11 @@ "rott/rt_ted.c" ], "github.com/finalburnneo/FBNeo@697e385c8285bfc528343b96d948ee3a4f9e24c5": [ + "src/burn/burn.cpp", + "src/burn/drv/neogeo/d_neogeo.cpp", + "src/burn/hiscore.cpp" + ], + "github.com/finalburnneo/FBNeo@c110542721d8674b6ff48e691ac032c040fecbba": [ "src/burn/burn.cpp", "src/burn/drv/atari/d_atarisy1.cpp", "src/burn/drv/channelf/d_channelf.cpp", @@ -3446,6 +3466,7 @@ "src/xenia/ui/vulkan/vulkan_instance.cc", "src/xenia/ui/windowed_app_main_win.cc" ], + "github.com/has207/xenia-edge@d881a29125978b4eea6bf44f6ed8c33f8aea8c33": [], "github.com/has207/xenia-edge@f23c14c717ef274516f29c9b5e97d6a9db28643d": [ ".ci/build-windows-cross.sh", "libretro/CMakeLists.txt", @@ -3678,6 +3699,7 @@ "github.com/iyzsong/onscripter-libretro@ec4ff1842d498faaa844d570becf561d088deea0": [ "libretro.cpp" ], + "github.com/iyzsong/syobonaction-libretro@792c40cb590bcc7286cf720b5fb85807082b534e": [], "github.com/izzy2lost/Super3@c410f82f771b33094b8c1cdbf96ef7c74ec9cff1": [ "Manual.txt", "README.md", @@ -3870,6 +3892,12 @@ "libretro.c" ], "github.com/libretro/FBNeo@32958a89abd645ed20387c8d31d21ede3bc48161": [ + "src/burn/burn.cpp", + "src/burn/drv/neogeo/d_neogeo.cpp", + "src/burn/hiscore.cpp", + "src/burner/libretro/libretro.cpp" + ], + "github.com/libretro/FBNeo@da14132bab6683980b1ad7e4d0d6fd47bf06dce7": [ "src/burn/burn.cpp", "src/burn/drv/atari/d_atarisy1.cpp", "src/burn/drv/channelf/d_channelf.cpp", @@ -4657,6 +4685,7 @@ "src/drivers/cvs.c", "src/drivers/decocass.c", "src/drivers/konamigx.c", + "src/drivers/namcos11.c", "src/drivers/neogeo.c", "src/drivers/nss.c", "src/drivers/pgm.c", @@ -7910,6 +7939,7 @@ "mupen64plus-video-gliden64/upstream/src/mupenplus/Config_mupenplus.cpp", "settings.gradle" ], + "github.com/mupen64plus/mupen64plus-core@3be6336a1de6f32ccc4f57bf80c988163fe6bf7a": [], "github.com/mupen64plus/mupen64plus-core@6dca4c15370ac3e2171ce7b31426695f8f39b460": [ "projects/unix/Makefile", "src/api/config.c", @@ -8535,6 +8565,7 @@ "video/external/src/decoder/openh264.rs", "web/Cargo.toml" ], + "github.com/sarah-walker-pcem/pcem@25b54e3ff1d51cbf2469b447e55bce7a589697e9": [], "github.com/sashkinbro/EmuCoreV@b71d8cf5397b1c6394e92eeb48427cd17e542971": [ "app/build.gradle.kts", "app/src/main/AndroidManifest.xml", @@ -9444,6 +9475,7 @@ "github.com/xenia-canary/xenia-canary@72c2afa3ba2b1b232f89de71577251d6edafb1bd": [ "src/xenia/emulator.cc" ], + "github.com/xenia-canary/xenia-canary@dc561e4df578c903087f9bdf56ff61d9ef9f2a93": [], "github.com/xenios-jp/XeniOS@87b176a078c316fde3adf67a217f0c44615b0e0d": [ "src/xenia/app/CMakeLists.txt", "src/xenia/app/game_compat_db.cc", @@ -9571,6 +9603,7 @@ "src/common/filesystem.c", "src/common/header/common.h" ], + "github.com/yuzu-emu-mirror/yuzu@15e6e48bef0216480661444a8d8b348c1cca47bb": [], "github.com/zelurker/raine@8e25d5d8907ae459181610912b0030c955683015": [ "source/emudx.c", "source/emumain.c", @@ -9704,6 +9737,7 @@ "src/jcv_db.c", "src/jollycv.c" ], + "gitlab.com/jgemu/nestopia@b3adbeb42482e979ab99e16bcd9a22480f13479f": [], "gitlab.com/parallel-launcher/parallel-launcher@8dc2a83770c57bc00e1b1fae0aed71f2120cc875": [ "src/core/retroarch.cpp", "src/core/rom-finder.cpp", @@ -9748,6 +9782,7 @@ "libretro/LibretroCore.cpp", "libretro/RomData.h" ], + "gitlab.com/recalbox/packages/standalone/rb5000@7464996aa0cb1be77abf495dc901117e1af36e02": [], "gitlab.com/recalbox/packages/standalone/rb5000@8c75a4b48a5222b0332d0508600f8ddc7ac1191c": [ "source/Data.cpp", "source/Engine.cpp", @@ -9785,6 +9820,7 @@ "src/graphics/Shader.cpp", "src/graphics/SpriteAnimationSet.cpp" ], + "sourceforge.net/dgen/dgen@1c196e63d3e35704a97a2cef60def9df3c9fd1ce": [], "voidpoint.io/terminx/eduke32@ec5824db81817866f70da326d3811bb0f52b3517": [ "GNUmakefile", "source/duke3d/src/common.cpp", diff --git a/wiki/tools.md b/wiki/tools.md index e0c6997b6..577296f43 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -525,7 +525,8 @@ prose that the same pass would have moved is not reported as blocked. Drop A moved pin also moves the links the site draws into the cited sources: a path is linked only where `website/src/data/source-paths.json` records it in the tree the profile pins. Refresh that record after a pin moves or a profile -is added, and commit it with the profiles: +is added, and commit it with the profiles. Every tree a run tries is recorded, +read or not, so `check_freshness.py` reports a pinned tree the record lacks: ```bash python scripts/sitesources.py --fetch