diff --git a/.gitignore b/.gitignore index f7774f60d..9aaaff995 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ site/ !bios/**/*.tmp *.log node_modules/ +# pip install -e . writes its metadata beside pyproject.toml. +*.egg-info/ +/build/ # Pages of the MkDocs build the site had before Astro Starlight: a clone may # still hold them. The Starlight project ignores its own generated folders. diff --git a/emulators/advanced_drastic.yml b/emulators/advanced_drastic.yml index 620b41e0a..019aa9ffa 100644 --- a/emulators/advanced_drastic.yml +++ b/emulators/advanced_drastic.yml @@ -7,6 +7,13 @@ author: "trngaje (hooks), Exophase (DraStic)" profiled_date: "2026-10-22" source_commit: "eaf41ab9bc8adcc15c010bf9517bc94c38e7bffd" pin_frozen: "KNULLI, the only platform that ships it, builds this revision (KNULLI package/emulators/advanced_drastic/advanced_drastic.mk:7 at tag 20260511). Later commits are not part of that build." +pin_build: + - platform: knulli + repo: "https://github.com/knulli-cfw/knulli-linux" + revision: "20260511" + path: "package/emulators/advanced_drastic/advanced_drastic.mk" + variable: ADVANCED_DRASTIC_VERSION + value: "eaf41ab9bc8adcc15c010bf9517bc94c38e7bffd" core_version: "r2.5.2.0" version_basis: kind: file diff --git a/emulators/advmame-1.4.yml b/emulators/advmame-1.4.yml index 490b994d7..db2d0f0d5 100644 --- a/emulators/advmame-1.4.yml +++ b/emulators/advmame-1.4.yml @@ -6,6 +6,13 @@ upstream: "https://www.mamedev.org" profiled_date: "2026-10-20" source_commit: "237ca5c898e51e1b7a70ce52bccf008aa722e924" pin_frozen: "RetroPie builds the advancemame-1.4 release tarball, whose src/ tree is this tag. Later commits are not part of that build." +pin_build: + - platform: retropie + repo: "https://github.com/RetroPie/RetroPie-Setup" + revision: "46128cd252cdecae852a4d0bafa2aaf9d03ab563" + path: "scriptmodules/emulators/advmame-1.4.sh" + variable: rp_module_repo + value: "file $__archive_url/advancemame-1.4.tar.gz" core_version: "advancemame-1.4" version_basis: kind: tag diff --git a/emulators/alephone.yml b/emulators/alephone.yml index d0dfaf1a6..6607f45e9 100644 --- a/emulators/alephone.yml +++ b/emulators/alephone.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/Aleph-One-Marathon/alephone" profiled_date: "2026-10-20" source_commit: "07a9c36aff403d143a0608815f725631a01e382c" pin_frozen: "RetroPie, the only platform that ships it, builds the release-20250829 tag at this commit (scriptmodules/ports/alephone.sh, _get_branch_alephone). Debian 10 and older get release-20250302." +pin_build: + - platform: retropie + repo: "https://github.com/RetroPie/RetroPie-Setup" + revision: "46128cd252cdecae852a4d0bafa2aaf9d03ab563" + path: "scriptmodules/ports/alephone.sh" + variable: version + value: "20250829" core_version: "release-20250829" version_basis: kind: tag diff --git a/emulators/azahar-sdl.yml b/emulators/azahar-sdl.yml index 126f775b3..0673125c2 100644 --- a/emulators/azahar-sdl.yml +++ b/emulators/azahar-sdl.yml @@ -5,6 +5,13 @@ source: "https://github.com/johndoe6345789/azahar_libretro" profiled_date: "2026-10-22" source_commit: "c8d39853c1ebacc5657e51d601713a09404ca8a7" pin_frozen: "KNULLI, the only platform that ships it, builds this revision (KNULLI package/emulators/azahar/azahar.mk:7 at tag 20260511). Later commits are not part of that build." +pin_build: + - platform: knulli + repo: "https://github.com/knulli-cfw/knulli-linux" + revision: "20260511" + path: "package/emulators/azahar/azahar.mk" + variable: AZAHAR_VERSION + value: "c8d39853c1ebacc5657e51d601713a09404ca8a7" core_version: "2122-rc1-178-gc8d3985" version_basis: kind: tag diff --git a/emulators/bennugd-monolithic.yml b/emulators/bennugd-monolithic.yml index 6f4e2a688..a3c6f490b 100644 --- a/emulators/bennugd-monolithic.yml +++ b/emulators/bennugd-monolithic.yml @@ -5,6 +5,13 @@ source: "https://github.com/christianhaitian/bennugd-monolithic" profiled_date: "2026-10-22" source_commit: "60ee3389efcf9b402d66035e87f33d17d70cbd83" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/bennugd-monolithic/package.mk:5 at v4.8). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/bennugd-monolithic/package.mk" + variable: PKG_VERSION + value: "60ee3389efcf9b402d66035e87f33d17d70cbd83" core_version: "1.0.0" version_basis: kind: file diff --git a/emulators/bermuda.yml b/emulators/bermuda.yml index 130dc2538..b2354f468 100644 --- a/emulators/bermuda.yml +++ b/emulators/bermuda.yml @@ -5,6 +5,13 @@ source: "https://github.com/cyxx/bermuda" profiled_date: "2026-10-22" source_commit: "cf6bdb68d53a1618967da565a44c931b15daf791" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/bermuda/package.mk:5 at v4.8). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/bermuda/package.mk" + variable: PKG_VERSION + value: "cf6bdb68d53a1618967da565a44c931b15daf791" core_version: "0.1.7" version_basis: kind: file diff --git a/emulators/ccleste.yml b/emulators/ccleste.yml index d2848132d..351257b8f 100644 --- a/emulators/ccleste.yml +++ b/emulators/ccleste.yml @@ -5,6 +5,13 @@ source: "https://github.com/lemon32767/ccleste" profiled_date: "2026-10-22" source_commit: "261d96f15af430b8111abc7a5250229246654f52" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/ccleste/package.mk:5 at v4.8). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/ccleste/package.mk" + variable: PKG_VERSION + value: "261d96f15af430b8111abc7a5250229246654f52" core_version: "v1.4.0" version_basis: kind: tag diff --git a/emulators/chocolate-doom.yml b/emulators/chocolate-doom.yml index b3f416c2e..5fffc5fad 100644 --- a/emulators/chocolate-doom.yml +++ b/emulators/chocolate-doom.yml @@ -5,6 +5,13 @@ source: "https://github.com/chocolate-doom/chocolate-doom" profiled_date: "2026-10-22" source_commit: "72d9e9318c4ccc61dab29bb9cc30bc8a9a93266d" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/chocolate-doom/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/chocolate-doom/package.mk" + variable: PKG_VERSION + value: "72d9e9318c4ccc61dab29bb9cc30bc8a9a93266d" core_version: "3.0.0" version_basis: kind: file diff --git a/emulators/daedalusx64.yml b/emulators/daedalusx64.yml index 526976ff1..9fa26a8ae 100644 --- a/emulators/daedalusx64.yml +++ b/emulators/daedalusx64.yml @@ -5,6 +5,13 @@ source: "https://github.com/DaedalusX64/daedalus" profiled_date: "2026-10-17" source_commit: "f17e9ed86f3806fadeb69abd29c9526ab2d4bd1b" pin_frozen: "ROCKNIX 20261001 builds this revision. Later upstream commits are not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/standalone/daedalusx64-sa/package.mk" + variable: PKG_VERSION + value: "f17e9ed86f3806fadeb69abd29c9526ab2d4bd1b" core_version: "1.1.8-837-gf17e9ed" version_basis: kind: tag diff --git a/emulators/dsperate.yml b/emulators/dsperate.yml index 9058a2dfa..f83d6f795 100644 --- a/emulators/dsperate.yml +++ b/emulators/dsperate.yml @@ -5,6 +5,13 @@ source: "https://github.com/beebono/DSperate" profiled_date: "2026-10-17" source_commit: "81b73832c2e68e2b10cb5850f0a6d393ffc6fbb7" pin_frozen: "ROCKNIX 20261001 ships the v2.0.1 release binaries, built by CI from the v2.0.1 tag at this commit." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/standalone/dsperate-sa/package.mk" + variable: PKG_VERSION + value: "2.0.1" core_version: "v2.0.1" version_basis: kind: tag diff --git a/emulators/duckstation-legacy.yml b/emulators/duckstation-legacy.yml index 35e5bf244..6c346f7f4 100644 --- a/emulators/duckstation-legacy.yml +++ b/emulators/duckstation-legacy.yml @@ -5,6 +5,13 @@ source: "https://github.com/stenzek/duckstation" profiled_date: "2026-10-22" source_commit: "56ac3d6c324ec4520dac433b071ecab6227fb459" pin_frozen: "Batocera builds this revision of 4 August 2023 as duckstation-legacy, its package marked not to be bumped, and KNULLI ships the same package." +pin_build: + - platform: batocera + repo: "https://github.com/batocera-linux/batocera.linux" + revision: "batocera-43.1" + path: "package/batocera/emulators/duckstation-legacy/duckstation-legacy.mk" + variable: DUCKSTATION_LEGACY_VERSION + value: "56ac3d6c324ec4520dac433b071ecab6227fb459" core_version: "0.1-5612-g56ac3d6" version_basis: kind: tag diff --git a/emulators/eightyone.yml b/emulators/eightyone.yml index bd2c20de4..33b94a5e9 100644 --- a/emulators/eightyone.yml +++ b/emulators/eightyone.yml @@ -11,6 +11,13 @@ version_basis: tag: "v1.2" distance: 0 pin_frozen: "Recalbox, the only platform that ships this core, builds its tag v1.2 (Recalbox package/libretro-eightyone/libretro-eightyone.mk:8 at tag 10.1.1)." +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/libretro-eightyone/libretro-eightyone.mk" + variable: LIBRETRO_EIGHTYONE_VERSION + value: "v1.2" display_name: "Sinclair - ZX81 / ZX Spectrum (EightyOne, Recalbox)" cores: ["eightyone"] built_by: [recalbox] diff --git a/emulators/exel-hence.yml b/emulators/exel-hence.yml index 84da03588..0539428f6 100644 --- a/emulators/exel-hence.yml +++ b/emulators/exel-hence.yml @@ -11,6 +11,13 @@ version_basis: tag: "v0.9" distance: 0 pin_frozen: "Recalbox, the only platform that ships this core, builds its tag v0.9 (Recalbox package/libretro-exel-hence/libretro-exel-hence.mk:9 at tag 10.1.1)." +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/libretro-exel-hence/libretro-exel-hence.mk" + variable: LIBRETRO_EXEL_HENCE_VERSION + value: "v0.9" display_name: "Exelvision - EXL 100 (Exel-Hence)" cores: - "exel-hence" diff --git a/emulators/fbalpha2019.yml b/emulators/fbalpha2019.yml index dc2100223..51d16bf37 100644 --- a/emulators/fbalpha2019.yml +++ b/emulators/fbalpha2019.yml @@ -7,6 +7,13 @@ profiled_date: "2026-10-17" source_commit: "0581797db6fdffd826086b053ced4b6b29bb6678" upstream_commit: "416948158d88ba17f87813419ddc3c1209ef68c0" pin_frozen: "ROCKNIX 20261001 builds this revision as fbalpha2019. Later commits on libretro/fbalpha are not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/libretro/fbalpha2019-lr/package.mk" + variable: PKG_VERSION + value: "0581797db6fdffd826086b053ced4b6b29bb6678" core_version: "v0.2.97.44" version_basis: kind: file diff --git a/emulators/fheroes2.yml b/emulators/fheroes2.yml index 75f787a71..da3a3ff72 100644 --- a/emulators/fheroes2.yml +++ b/emulators/fheroes2.yml @@ -5,6 +5,13 @@ source: "https://github.com/ihhub/fheroes2" profiled_date: "2026-10-22" source_commit: "35ca4c915be00098547d5050412a325dd55cfd7f" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/fheroes2/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/fheroes2/package.mk" + variable: PKG_VERSION + value: "35ca4c915be00098547d5050412a325dd55cfd7f" core_version: "1.0.11" version_basis: kind: file diff --git a/emulators/flycast2021.yml b/emulators/flycast2021.yml index 7f6257b87..33b357aad 100644 --- a/emulators/flycast2021.yml +++ b/emulators/flycast2021.yml @@ -7,6 +7,13 @@ profiled_date: "2026-10-17" source_commit: "603814c9f73b773c455d9a497f389d2f93a257fd" upstream_commit: "4c293f306bc16a265c2d768af5d0cea138426054" pin_frozen: "ROCKNIX 20261001 builds this revision. No branch of metallic77/flycast holds it any more: master was put back on the libretro merge base 4c293f30, so HEAD is the code before the fork's own commits." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/libretro/flycast2021-lr/package.mk" + variable: PKG_VERSION + value: "603814c9f73b773c455d9a497f389d2f93a257fd" core_version: "0.1" version_basis: kind: file diff --git a/emulators/flycast2021le.yml b/emulators/flycast2021le.yml index e66242c0f..2ced2bd01 100644 --- a/emulators/flycast2021le.yml +++ b/emulators/flycast2021le.yml @@ -7,6 +7,13 @@ profiled_date: "2026-10-22" source_commit: "bf93aece5db7050667075a1107ab380856b44a23" upstream_commit: "4c293f306bc16a265c2d768af5d0cea138426054" pin_frozen: "AmberELEC, the only platform that ships it, builds this revision (AmberELEC packages/games/libretro/flycast2021le/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: amberelec + repo: "https://github.com/AmberELEC/AmberELEC" + revision: "a21975bfb1a3af68f66e07a931b994b523e1e6e2" + path: "packages/games/libretro/flycast2021le/package.mk" + variable: PKG_VERSION + value: "bf93aece5db7050667075a1107ab380856b44a23" core_version: "0.1" version_basis: kind: file diff --git a/emulators/gsplus-legacy.yml b/emulators/gsplus-legacy.yml index f63659cd7..dc3c8a6cd 100644 --- a/emulators/gsplus-legacy.yml +++ b/emulators/gsplus-legacy.yml @@ -7,6 +7,13 @@ source: "https://github.com/digarok/gsplus-legacy-archive" profiled_date: "2026-10-20" source_commit: "480572054518112647c8fae5d7ea7046a6d6ecfb" pin_frozen: "Recalbox builds this commit (Recalbox package/gsplus/gsplus.mk:8-9 at tag 10.1.1). Batocera builds the applemu fork and RetroBat its release build, as the notes say." +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/gsplus/gsplus.mk" + variable: GSPLUS_VERSION + value: "480572054518112647c8fae5d7ea7046a6d6ecfb" core_version: "0.14" version_basis: kind: file diff --git a/emulators/hode.yml b/emulators/hode.yml index 88c659e9e..a8d1ec333 100644 --- a/emulators/hode.yml +++ b/emulators/hode.yml @@ -5,6 +5,13 @@ source: "https://github.com/usineur/hode" profiled_date: "2026-10-22" source_commit: "59f3c466923c6b8d49e29176ff697a91165e7efc" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/hodesdl/package.mk:6 at v4.8). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/hodesdl/package.mk" + variable: PKG_VERSION + value: "59f3c466923c6b8d49e29176ff697a91165e7efc" core_version: "0.2.9f" version_basis: kind: file diff --git a/emulators/love-0.10.2.yml b/emulators/love-0.10.2.yml index ca41ff412..9fc298b50 100644 --- a/emulators/love-0.10.2.yml +++ b/emulators/love-0.10.2.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/love2d/love" profiled_date: "2026-10-20" source_commit: "9e139779956ddba89333eece8104bd7b1b4ff2e7" pin_frozen: "RetroPie, the only platform that ships it, builds the 0.10.2 tag at this commit (RetroPie scriptmodules/ports/love-0.10.2.sh:16). Later releases are packaged separately as love." +pin_build: + - platform: retropie + repo: "https://github.com/RetroPie/RetroPie-Setup" + revision: "46128cd252cdecae852a4d0bafa2aaf9d03ab563" + path: "scriptmodules/ports/love-0.10.2.sh" + variable: rp_module_repo + value: "git https://github.com/love2d/love 0.10.2" core_version: "0.10.2" version_basis: kind: tag diff --git a/emulators/lzdoom.yml b/emulators/lzdoom.yml index 8ca3a0e57..0aa911f89 100644 --- a/emulators/lzdoom.yml +++ b/emulators/lzdoom.yml @@ -5,6 +5,19 @@ source: "https://github.com/drfrag666/lzdoom" profiled_date: "2026-10-20" source_commit: "4ce19a12164ebd6802b7fb2f94d700a6e736d0fb" pin_frozen: "The platforms that ship it run the 3.88 line: RetroPie builds tag 3.88b (RetroPie scriptmodules/ports/lzdoom.sh:15) and AmberELEC a fork of the g3.3mgw branch that still reports 3.88b (AmberELEC packages/games/emulators/lzdoom/package.mk:5). The l4.x releases are a GZDoom 4 based line that no platform ships." +pin_build: + - platform: retropie + repo: "https://github.com/RetroPie/RetroPie-Setup" + revision: "46128cd252cdecae852a4d0bafa2aaf9d03ab563" + path: "scriptmodules/ports/lzdoom.sh" + variable: rp_module_repo + value: "git https://github.com/drfrag666/gzdoom 3.88b" + - platform: amberelec + repo: "https://github.com/AmberELEC/AmberELEC" + revision: "a21975bfb1a3af68f66e07a931b994b523e1e6e2" + path: "packages/games/emulators/lzdoom/package.mk" + variable: PKG_VERSION + value: "8e12e62164ea1ba52625c63bc0f1c9e0076bff4c" core_version: "3.88b" version_basis: kind: tag diff --git a/emulators/mame0258.yml b/emulators/mame0258.yml index 76bc45c4f..3df4ed857 100644 --- a/emulators/mame0258.yml +++ b/emulators/mame0258.yml @@ -7,6 +7,13 @@ profiled_date: '2026-10-22' source_commit: dc1a96537089d6d4ecd01d9886b21061471c70c0 upstream_commit: 2e0aa823508663c4a27164be43467d1ef740cdcd pin_frozen: 'Recalbox, the only platform that ships it, builds this revision (Recalbox package/libretro-mame0258/libretro-mame0258.mk:8 at tag 10.1.1). Later commits are not part of that build.' +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/libretro-mame0258/libretro-mame0258.mk" + variable: LIBRETRO_MAME0258_VERSION + value: "dc1a96537089d6d4ecd01d9886b21061471c70c0" core_version: "0.258" version_basis: kind: file diff --git a/emulators/mame0278.yml b/emulators/mame0278.yml index 8ba5f4673..f37e94c1f 100644 --- a/emulators/mame0278.yml +++ b/emulators/mame0278.yml @@ -7,6 +7,13 @@ profiled_date: '2026-10-22' source_commit: fba50741006fa3e27e1897f589d2048a505b8fd6 upstream_commit: eb05584a2b8d98d5af5587244c9cb7dcccf660fd pin_frozen: 'Recalbox, the only platform that ships it, builds this revision (Recalbox package/libretro-mame0278/libretro-mame0278.mk:8 at tag 10.1.1). Later commits are not part of that build.' +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/libretro-mame0278/libretro-mame0278.mk" + variable: LIBRETRO_MAME0278_VERSION + value: "fba50741006fa3e27e1897f589d2048a505b8fd6" core_version: "0.278" version_basis: kind: file diff --git a/emulators/nanoboyadvance.yml b/emulators/nanoboyadvance.yml index bbd61b392..0e113e0f4 100644 --- a/emulators/nanoboyadvance.yml +++ b/emulators/nanoboyadvance.yml @@ -5,6 +5,13 @@ source: "https://github.com/nba-emu/NanoBoyAdvance" profiled_date: "2026-10-17" source_commit: "3bb6f478f977dbfd3106508536e5fbce90d1898b" pin_frozen: "ROCKNIX 20261001 builds this revision of the SDL2 frontend (v1.6 plus four commits). Later upstream work is not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/standalone/nanoboyadvance-sa/package.mk" + variable: PKG_VERSION + value: "3bb6f478f977dbfd3106508536e5fbce90d1898b" core_version: "v1.6-4-g3bb6f47" version_basis: kind: tag diff --git a/emulators/piemu.yml b/emulators/piemu.yml index f0b5bdef3..12e1ec8b4 100644 --- a/emulators/piemu.yml +++ b/emulators/piemu.yml @@ -7,6 +7,13 @@ profiled_date: "2026-10-22" source_commit: "0785214f13746bdfd709b1909f007e213b470ba1" upstream_commit: "23208626e16e53b73fd21a2fbd0c429338625dc4" pin_frozen: "AmberELEC, the only platform that ships it, builds this revision (AmberELEC packages/games/emulators/piemu/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: amberelec + repo: "https://github.com/AmberELEC/AmberELEC" + revision: "a21975bfb1a3af68f66e07a931b994b523e1e6e2" + path: "packages/games/emulators/piemu/package.mk" + variable: PKG_VERSION + value: "0785214f13746bdfd709b1909f007e213b470ba1" core_version: "20030430" version_basis: kind: file diff --git a/emulators/ppsspp-1.5.4.yml b/emulators/ppsspp-1.5.4.yml index 1ba7ad930..d5f8a3d16 100644 --- a/emulators/ppsspp-1.5.4.yml +++ b/emulators/ppsspp-1.5.4.yml @@ -5,6 +5,13 @@ source: "https://github.com/hrydgard/ppsspp" profiled_date: "2026-10-20" source_commit: "a1e74d0d4f89ba3fa2d4fe64bac7a0fa16fc146f" pin_frozen: "RetroPie, the only platform that ships it, builds the v1.5.4 tag at this commit (RetroPie scriptmodules/emulators/ppsspp-1.5.4.sh:16). Later releases are packaged separately as ppsspp." +pin_build: + - platform: retropie + repo: "https://github.com/RetroPie/RetroPie-Setup" + revision: "46128cd252cdecae852a4d0bafa2aaf9d03ab563" + path: "scriptmodules/emulators/ppsspp-1.5.4.sh" + variable: rp_module_repo + value: "git https://github.com/hrydgard/ppsspp.git v1.5.4" core_version: "v1.5.4" version_basis: kind: tag diff --git a/emulators/rb5000.yml b/emulators/rb5000.yml index 01d9064db..637766890 100644 --- a/emulators/rb5000.yml +++ b/emulators/rb5000.yml @@ -10,6 +10,13 @@ source_commit: standalone: "8c75a4b48a5222b0332d0508600f8ddc7ac1191c" libretro: "7464996aa0cb1be77abf495dc901117e1af36e02" pin_frozen: "Recalbox, the only platform that ships the standalone build, builds its tag v1.1 at this standalone revision (Recalbox package/rb5000/rb5000.mk:7). Later commits are not part of that build. The libretro pin is the HEAD of libretro-rb5000." +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/rb5000/rb5000.mk" + variable: RB5000_VERSION + value: "v1.1" core_version: standalone: "v1.1" libretro: "v0.9" diff --git a/emulators/reicast.yml b/emulators/reicast.yml index 975bcfa4c..eb7784e7e 100644 --- a/emulators/reicast.yml +++ b/emulators/reicast.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/skmp/reicast-emulator" profiled_date: "2026-10-17" source_commit: "ad61f95dd6b4c6218846a25de6194550311a5d28" pin_frozen: "Recalbox 10.1.1 builds this revision (package/reicast/reicast.mk REICAST_VERSION). Later commits on reicast-emulator are not part of that build." +pin_build: + - platform: recalbox + repo: "https://gitlab.com/recalbox/recalbox" + revision: "10.1.1" + path: "package/reicast/reicast.mk" + variable: REICAST_VERSION + value: "ad61f95dd6b4c6218846a25de6194550311a5d28" core_version: "r8.1-243-gad61f95" version_basis: kind: tag diff --git a/emulators/rigelengine.yml b/emulators/rigelengine.yml index 002d4996e..1e6996c25 100644 --- a/emulators/rigelengine.yml +++ b/emulators/rigelengine.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/lethal-guitar/RigelEngine" profiled_date: "2026-10-22" source_commit: "f05996f9b3ad3b3ea5bb818e49e7977636746343" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/rigelengine/package.mk:5 at v4.8). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/rigelengine/package.mk" + variable: PKG_VERSION + value: "f05996f9b3ad3b3ea5bb818e49e7977636746343" core_version: "0.9.1" version_basis: kind: file diff --git a/emulators/supermariowar.yml b/emulators/supermariowar.yml index d2ce65c99..6f3d87b05 100644 --- a/emulators/supermariowar.yml +++ b/emulators/supermariowar.yml @@ -5,6 +5,13 @@ source: "https://github.com/mmatyas/supermariowar" profiled_date: "2026-10-22" source_commit: "ed60ed45e296ff88a8883cbc51f5bf5aed880636" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/supermariowar/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/supermariowar/package.mk" + variable: PKG_VERSION + value: "ed60ed45e296ff88a8883cbc51f5bf5aed880636" core_version: "2.0.0" version_basis: kind: file diff --git a/emulators/supermodel-legacy.yml b/emulators/supermodel-legacy.yml index a501a3883..ea183429c 100644 --- a/emulators/supermodel-legacy.yml +++ b/emulators/supermodel-legacy.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/trzy/Supermodel" profiled_date: "2026-10-17" source_commit: "0eef09ba2b74e48fd5cfd388ee69a3743fd5f618" pin_frozen: "Batocera's supermodel-legacy package builds this revision, held before the renderer moved to OpenGL 4.1. Recaling would repoint these refs at code this build never had." +pin_build: + - platform: batocera + repo: "https://github.com/batocera-linux/batocera.linux" + revision: "batocera-43.1" + path: "package/batocera/emulators/supermodel-legacy/supermodel-legacy.mk" + variable: SUPERMODEL_LEGACY_VERSION + value: "0eef09ba2b74e48fd5cfd388ee69a3743fd5f618" core_version: "0.3a-WIP" version_basis: kind: file diff --git a/emulators/supersnes9x.yml b/emulators/supersnes9x.yml index bb8966d61..d6f387124 100644 --- a/emulators/supersnes9x.yml +++ b/emulators/supersnes9x.yml @@ -6,6 +6,13 @@ upstream: "https://github.com/snes9xgit/snes9x" profiled_date: "2026-10-17" source_commit: "691e47b40fb38fa522a5471462344a2f0206b7a8" pin_frozen: "ROCKNIX 20261001 builds this revision. Later commits on shanytc/snes9x rework the Super Famicom Box loader and are not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/libretro/supersnes9x-lr/package.mk" + variable: PKG_VERSION + value: "691e47b40fb38fa522a5471462344a2f0206b7a8" core_version: "1.63.29" version_basis: kind: file diff --git a/emulators/supertux.yml b/emulators/supertux.yml index c05f1b9d6..5b086f101 100644 --- a/emulators/supertux.yml +++ b/emulators/supertux.yml @@ -5,6 +5,13 @@ source: "https://github.com/SuperTux/supertux" profiled_date: "2026-10-22" source_commit: "37cddf24a4abfdec7931a84afd9e87fb8ea18767" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision, v0.6.3 (EmuELEC packages/sx05re/emuelec-ports/supertux/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/supertux/package.mk" + variable: PKG_VERSION + value: "37cddf24a4abfdec7931a84afd9e87fb8ea18767" core_version: "v0.6.3-1020-g37cddf2" version_basis: kind: tag diff --git a/emulators/supertuxkart.yml b/emulators/supertuxkart.yml index 9fe306c0f..f2d9f99b5 100644 --- a/emulators/supertuxkart.yml +++ b/emulators/supertuxkart.yml @@ -5,6 +5,13 @@ source: "https://github.com/supertuxkart/stk-code" profiled_date: "2026-10-22" source_commit: "fab79150111944c9e33396db68eb5de84f2efeaa" pin_frozen: "EmuELEC, the only platform that ships it, builds this revision (EmuELEC packages/sx05re/emuelec-ports/supertuxkart/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: emuelec + repo: "https://github.com/EmuELEC/EmuELEC" + revision: "v4.8" + path: "packages/sx05re/emuelec-ports/supertuxkart/package.mk" + variable: PKG_VERSION + value: "fab79150111944c9e33396db68eb5de84f2efeaa" core_version: "preview-582-gfab7915" version_basis: kind: tag diff --git a/emulators/touchhle.yml b/emulators/touchhle.yml index d588e5190..442667f59 100644 --- a/emulators/touchhle.yml +++ b/emulators/touchhle.yml @@ -5,6 +5,13 @@ source: "https://github.com/touchHLE/touchHLE" profiled_date: "2026-10-17" source_commit: "3c5850585ba55615b17a58c58331b6d6f52d4a9d" pin_frozen: "ROCKNIX 20261001 builds this revision. Later upstream commits are not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/standalone/touchhle-sa/package.mk" + variable: PKG_VERSION + value: "3c5850585ba55615b17a58c58331b6d6f52d4a9d" core_version: "0.2.3" version_basis: kind: file diff --git a/emulators/vice-standalone.yml b/emulators/vice-standalone.yml index 612ac8f1f..c4a61a000 100644 --- a/emulators/vice-standalone.yml +++ b/emulators/vice-standalone.yml @@ -7,6 +7,13 @@ logo: "https://raw.githubusercontent.com/VICE-Team/svn-mirror/main/vice/data/com profiled_date: "2026-10-24" source_commit: "81546e60c222fe4aef750b890b0dbe3a5ac280d8" pin_frozen: "ROCKNIX, the only platform that ships it, builds this revision (ROCKNIX projects/ROCKNIX/packages/emulators/standalone/vice-sa/package.mk:5). Later commits are not part of that build." +pin_build: + - platform: rocknix + repo: "https://github.com/ROCKNIX/distribution" + revision: "20261001" + path: "projects/ROCKNIX/packages/emulators/standalone/vice-sa/package.mk" + variable: PKG_VERSION + value: "3.8" core_version: "3.8" version_basis: kind: file diff --git a/pyproject.toml b/pyproject.toml index 95e707656..bfb24f9c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,14 @@ +# Installing the project installs its dependencies and nothing else: +# `pip install -e ".[dev]"` gives a contributor every tool the tests use. +# scripts/ and tests/ run from the checkout, so no package is built from them. +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [] +py-modules = [] + [project] name = "retrobios" version = "1.0.0" diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 661ec316b..9ca4a8418 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -10,6 +10,19 @@ "type" ], "allOf": [ + { + "comment": "A build reference documents why a pin is held: it never stands without the pin_frozen that says so.", + "if": { + "required": [ + "pin_build" + ] + }, + "then": { + "required": [ + "pin_frozen" + ] + } + }, { "comment": "An alias profile points at another profile and carries no files of its own.", "if": { @@ -991,6 +1004,43 @@ "type": "string", "description": "Why source_commit is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag: comparing it to HEAD says nothing and recaling would repoint its refs at code the build it documents never had." }, + "pin_build": { + "type": "array", + "minItems": 1, + "description": "The build file each platform that ships a pin_frozen build reads its revision from, so check_freshness can read it again where the platform's YAML now stands and say when the platform builds another revision.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["platform", "repo", "revision", "path", "variable", "value"], + "properties": { + "platform": { + "type": "string", + "description": "Registry slug of the platform (platforms/.yml)." + }, + "repo": { + "type": "string", + "format": "uri", + "description": "The platform's own repository, the one its YAML is scraped from." + }, + "revision": { + "type": "string", + "description": "Tag or commit of that repository the build file was read at." + }, + "path": { + "type": "string", + "description": "The build file, from the repository root (package.mk, a Buildroot .mk, a RetroPie scriptmodule)." + }, + "variable": { + "type": "string", + "description": "The variable the file sets to the revision it builds (PKG_VERSION, _VERSION, rp_module_repo)." + }, + "value": { + "type": "string", + "description": "What the variable held at revision, quotes and trailing comment removed." + } + } + } + }, "pin_follows": { "type": "string", "description": "Why source_commit follows the branch tip although a single platform ships the profile: that platform builds the tip, cited by its build file. Without it or pin_frozen, a bump refuses a profile one platform ships." diff --git a/scripts/artifacts.py b/scripts/artifacts.py index 82a120302..7cbfebc17 100644 --- a/scripts/artifacts.py +++ b/scripts/artifacts.py @@ -12,6 +12,7 @@ import contextlib import json import os import tempfile +import threading import re @@ -62,12 +63,26 @@ def write_if_changed( return True +_UMASK_LOCK = threading.Lock() + + +def process_umask() -> int: + """The process umask, read without letting a second thread see it at 0. + + Reading the umask sets it, so it is put back at once. Two threads doing + that unguarded interleave as A sets 0 and reads 022, B sets 0 and reads 0, + A restores 022, B restores 0: the process keeps umask 0 and every file it + creates afterwards is world-writable. + """ + with _UMASK_LOCK: + umask = os.umask(0) + os.umask(umask) + return umask + + def plain_file_mode() -> int: """The mode open() gives a new file: 0o666 less the process umask.""" - # Reading the umask sets it, so it is put back at once. - umask = os.umask(0) - os.umask(umask) - return 0o666 & ~umask + return 0o666 & ~process_umask() def set_plain_mode(path: str | os.PathLike) -> None: diff --git a/scripts/auto_fetch.py b/scripts/auto_fetch.py index 609160a6c..8db87d9ac 100644 --- a/scripts/auto_fetch.py +++ b/scripts/auto_fetch.py @@ -107,7 +107,10 @@ def find_missing(config: dict, db: dict) -> list[dict]: "destination": file_entry.get("destination", name), }) for u in result["undeclared_files"]: - if u.get("in_repo") or not u.get("required"): + # A file installed with its emulator is not an acquisition target: + # verify leaves it out of the missing count, and bundled_only.py + # --strict refuses a copy that only bundled entries reach. + if u.get("in_repo") or not u.get("required") or u.get("bundled"): continue missing.append({ "name": u["name"], diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index ec799b866..fac97bc71 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -524,7 +524,7 @@ def check_native( ) if reason: reason += ( - f", export_native.py --platform {name} --refresh-cache" + f", python scripts/export_native.py --platform {name} --refresh-cache" " fetches it, then the export is written again" ) else: @@ -532,7 +532,7 @@ def check_native( name, exporter_class, truth_dir, platforms_dir, cache_dir, published ) if reason: - reason += f", export_native.py --platform {name} writes it again" + reason += f", python scripts/export_native.py --platform {name} writes it again" findings.append( Finding( "native", @@ -974,7 +974,7 @@ def check_coreinfo( if recorded is None: findings.append(Finding( "coreinfo", "reviewed .info", UNKNOWN, - detail=f"no {record_path.name}: run check_freshness.py --record-coreinfo", + detail=f"no {record_path.name}: run python scripts/check_freshness.py --record-coreinfo", )) return findings findings.extend( @@ -982,7 +982,7 @@ def check_coreinfo( "coreinfo", name, STALE, local=(recorded.get(name) or "not recorded")[:12], remote=blobs[name][:12], detail=(".info changed since reviewed: read its firmware, then " - f"check_freshness.py --record-coreinfo {name}"), + f"python scripts/check_freshness.py --record-coreinfo {name}"), ) for name in coreinfo_drift(blobs, recorded, profiles) ) @@ -1456,11 +1456,122 @@ def check_source_paths(profiles: dict) -> Finding: "source-paths.json", STALE if problems else OK, local=f"{len(recorded)} trees", - detail=(", ".join(problems) + ": run scripts/sitesources.py --fetch") if problems + detail=(", ".join(problems) + ": run python scripts/sitesources.py --fetch") if problems else "every pinned tree is recorded, and only those", ) +def build_variable(lines: list[str], variable: str) -> str | None: + """The value the first assignment to ``variable`` gives in a build file. + + Read as a Buildroot .mk (`X_VERSION = sha`), a LibreELEC package.mk + (`PKG_VERSION="sha"`) or a RetroPie scriptmodule (`rp_module_repo="git + URL tag"`, `local version="20250829"`): quotes and a trailing comment go, + the rest is compared as written. + """ + assignment = re.compile( + rf"^\s*(?:local\s+|export\s+)?{re.escape(variable)}\s*[:?+]?=\s*(.*)$" + ) + for line in lines: + match = assignment.match(line) + if not match: + continue + value = match.group(1).strip() + if value[:1] in "\"'": + end = value.find(value[0], 1) + return value[1:end] if end > 0 else value[1:] + return value.split(" #", 1)[0].strip() + return None + + +def pin_build_revision(reference: dict, platforms_dir: Path) -> str: + """The revision of the platform repository its YAML was last scraped at, + '' when the platform is not registered or reads another repository.""" + path = platforms_dir / f"{reference.get('platform', '')}.yml" + repo = upstream.parse_repo(str(reference.get("repo") or "")) + if repo is None or not path.is_file(): + return "" + with path.open(encoding="utf-8") as fh: + source = str((yaml_load(fh) or {}).get("source") or "") + if f"/{repo.slug.lower()}/" not in source.lower() + "/": + return "" + return export_native.source_revision(source) + + +def check_pin_builds( + profiles: dict, platforms_dir: Path, cache_dir: str, offline: bool, +) -> list[Finding]: + """Each `pin_build:` read again where the platform now stands. + + A pin_frozen pin is held because one platform builds that revision, and + nothing read the build file again: Recalbox moving mame0278 to another + commit left the profile describing a build no platform ships. The file + is read at the revision the platform's own YAML was scraped at, or at + the repository's branch head for a platform this repository does not + scrape, and its variable compared with the value the profile recorded. + """ + findings: list[Finding] = [] + unreferenced = sorted( + name for name, profile in profiles.items() + if profile.get("pin_frozen") and not profile.get("pin_build") + ) + if unreferenced: + findings.append(Finding( + "profiles", "pin_frozen without pin_build", SKIPPED, + local=f"{len(unreferenced)} profiles", + detail="no platform build file names their revision: " + + ", ".join(unreferenced[:SHOWN_NAMES]) + + (", ..." if len(unreferenced) > SHOWN_NAMES else ""), + items=unreferenced, + )) + for name, profile in sorted(profiles.items()): + for reference in profile.get("pin_build") or []: + subject = f"{name} ({reference['platform']})" + recorded = str(reference["value"]) + revision = pin_build_revision(reference, platforms_dir) + if revision == str(reference["revision"]): + findings.append(Finding( + "profiles", subject, OK, local=recorded[:12], + detail=f"read where platforms/{reference['platform']}.yml is", + )) + continue + repo = upstream.parse_repo(str(reference["repo"])) + try: + at = revision or upstream.resolve_head(repo, cache_dir, offline) + lines = upstream.fetch_file(repo, at, reference["path"], cache_dir, offline) \ + if at else None + except upstream.NotCachedError: + findings.append(Finding("profiles", subject, SKIPPED, local=recorded[:12], + detail="offline, the build file is not cached")) + continue + except upstream.UpstreamError as exc: + findings.append(Finding("profiles", subject, UNKNOWN, local=recorded[:12], + detail=str(exc))) + continue + if not at: + findings.append(Finding("profiles", subject, UNKNOWN, local=recorded[:12], + detail=f"{repo.slug} answered no branch head")) + continue + current = None if lines is None else build_variable(lines, reference["variable"]) + if current == recorded: + findings.append(Finding( + "profiles", subject, OK, local=recorded[:12], remote=at[:12], + detail=f"{reference['path']} still builds it", + )) + continue + findings.append(Finding( + "profiles", subject, STALE, local=recorded[:12], + remote=(current or "gone")[:12], + detail=( + f"{reference['path']} is gone at {at[:12]}" if lines is None + else f"{reference['variable']} is no longer set there" if current is None + else f"{reference['platform']} now builds {current} " + f"({reference['path']} at {at[:12]}): the pin follows the build" + ), + )) + return findings + + def check_profiles(run: bool, offline: bool) -> list[Finding]: if not run: return [ @@ -1468,7 +1579,7 @@ def check_profiles(run: bool, offline: bool) -> list[Finding]: "profiles", "profile_sync", SKIPPED, - detail="pass --profiles, or run profile_sync.py --all --triage", + detail="pass --profiles, or run python scripts/profile_sync.py --all --triage", ) ] cmd = [ @@ -1708,6 +1819,7 @@ def main() -> int: findings.extend(check_ci(args.cache_dir, args.offline)) if "profiles" in areas: findings.append(check_source_paths(profiles)) + findings.extend(check_pin_builds(profiles, platforms_dir, args.cache_dir, args.offline)) findings.extend(check_profiles(args.profiles, args.offline)) if args.json: diff --git a/scripts/commit_rules.py b/scripts/commit_rules.py index 1dc2bf6fa..356b3a871 100644 --- a/scripts/commit_rules.py +++ b/scripts/commit_rules.py @@ -21,6 +21,14 @@ REFUSED_WORDS = re.compile( r"comprehensive|implement)", re.IGNORECASE, ) +# A numbered pointer at a private review ("audit 24", "review 3", "round 5") +# names notes no clone holds. tests/test_prose_style.py refuses the same tags +# in tracked files, and history cannot be rewritten once pushed. +REVIEW_TAG = re.compile( + r"\b(audit|review|round|batch|finding)s? #?\d+|\baudit (batch|findings?)\b" + r"|[a-z]+_audit_\d+", + re.IGNORECASE, +) CO_AUTHOR = re.compile(r"^co-authored-by", re.IGNORECASE | re.MULTILINE) @@ -35,6 +43,8 @@ def problems(message: str) -> list[str]: text = "\n".join(line for line in message.splitlines() if not line.startswith("#")) if REFUSED_WORDS.search(text): found.append("the message uses a word this repository's history does not carry") + if REVIEW_TAG.search(text): + found.append("say what the change does, not which review asked for it") if CO_AUTHOR.search(message): found.append("no Co-Authored-By trailer here") return found diff --git a/scripts/common.py b/scripts/common.py index d3e445fec..64b28fe08 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -691,7 +691,7 @@ def load_target_config( # --target but have no --list-targets of their own. + suggest_names( [target], sorted(alias_index), - f"verify.py --platform {platform_name} --list-targets", len(targets), + f"python scripts/verify.py --platform {platform_name} --list-targets", len(targets), ) ) @@ -1870,12 +1870,12 @@ def select_emulator_profiles( all_profiles: dict, standalone: bool = False, ) -> list[tuple[str, dict]]: - """Resolve named profiles, refusing the ones that answer for others. + """Resolve named profiles, refusing the ones that answer for no core. - An alias is the same binary under another name and a launcher only - starts an emulator, so neither has requirements of its own: naming one - is a question about the wrong profile, and the error says which one to - ask instead. Raising rather than exiting lets the verifier stop and the + An alias is the same binary under another name, so naming one selects + the profile it is an alias of, said on stderr. A launcher only starts an + emulator and has no requirements of its own: naming one is a question + about the wrong profile, and the error says so. Raising rather than exiting lets the verifier stop and the builder return empty-handed from one rule. """ selected: list[tuple[str, dict]] = [] @@ -1916,11 +1916,17 @@ def select_emulator_profiles( ), None, ) - raise ProfileSelectionError( - f"{name} is an alias of {target}: use --emulator {target}" - if target - else f"{name} is an alias with no profile of its own to name" - ) + if not target or target not in all_profiles: + raise ProfileSelectionError( + f"{name} is an alias with no profile of its own to name" + ) + # The name a user reads in RetroArch or on the buildbot: the tool + # knows the profile, so it says which and carries on. + print(f"{name} is {target}: using its profile", file=sys.stderr) + name, profile = target, all_profiles[target] + kind = profile.get("type", "libretro") + if any(key == name for key, _ in selected): + continue if kind == "launcher": raise ProfileSelectionError( f"{name} is a launcher: use the emulator it launches" @@ -3250,26 +3256,55 @@ def build_target_cores_cache( """ cache: dict[str, set[str] | None] = {} skip: list[str] = [] + notes: list[str] = [] for p in platforms: try: cache[p] = load_target_config(p, target, platforms_dir) except FileNotFoundError: if not is_all: raise - print(f"INFO: Skipping {p}: no hardware targets") + notes.append(f"INFO: Skipping {p}: no hardware targets") skip.append(p) except ValueError as e: if is_all: - print(f"INFO: Skipping {p}: {e}") + notes.append(f"INFO: Skipping {p}: {e}") skip.append(p) else: raise kept = [p for p in platforms if p not in skip] if is_all and platforms and not kept: - raise ValueError(f"unknown target '{target}': no platform has it") + # The seventeen per-platform lines buried the one suggestion that + # mattered: the refusal names the closest target across them all. + raise ValueError(_unknown_target_message(target, platforms, platforms_dir)) + for note in notes: + print(note) return cache, kept +def _unknown_target_message(target: str, platforms: list[str], platforms_dir: str) -> str: + """A target no platform has, with the closest names any of them knows.""" + import difflib # noqa: PLC0415 + + targets_dir = os.path.join(platforms_dir, "targets") + owners: dict[str, list[str]] = {} + for platform in platforms: + target_file = os.path.join(targets_dir, f"{platform}.yml") + if not os.path.exists(target_file): + continue + with open(target_file) as f: + targets = (yaml_load(f) or {}).get("targets", {}) + overrides = _target_overrides(platform, targets_dir) + for name in _target_alias_index(targets, overrides): + owners.setdefault(name, []).append(platform) + message = f"unknown target '{target}': no platform has it." + close = difflib.get_close_matches(target, sorted(owners), n=3, cutoff=0.6) + if close: + named = ", ".join(f"{n} ({', '.join(sorted(owners[n]))})" for n in close) + return f"{message} Did you mean: {named}?" + return (f"{message} python scripts/verify.py --platform --list-targets " + "lists the targets of one platform.") + + # Re-exported so the existing call sites keep working while the # modules above become the place to import from. from safeparse import ( # noqa: E402,F401 diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index 5ceb69f2d..d2e259928 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -375,6 +375,7 @@ class EntryCategories: # entry with a proof walks its reader's whole profile, and a MAME # profile's archive members were hashed again for each. self._accepted: dict[int, set[str]] = {} + self._entry_index: dict[tuple[str, str], tuple[dict, dict]] = {} # One index per core and per kind, so a proof names its reader. self._readers: list[tuple[str, str, ReadEvidence]] = [] for key in sorted(examined): @@ -418,6 +419,24 @@ class EntryCategories: accepted = self._accepted[id(profile_entry)] = _entry_hashes(profile_entry) return accepted + def _index_of(self, key: str, kind: str) -> tuple[dict, dict]: + """A profile's entries of one kind by casefolded name and by accepted + hash, each list in file order. Built once per profile and kind: a + scan per platform entry cost 2 s of verify --all on MAME alone.""" + index = self._entry_index.get((key, kind)) + if index is None: + by_name: dict[str, list[tuple[int, dict]]] = {} + by_hash: dict[str, list[tuple[int, dict]]] = {} + for position, f in enumerate(self._profiles[key].get("files") or []): + if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA): + continue + for n in {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}: + by_name.setdefault(n, []).append((position, f)) + for value in self._accepted_by(f): + by_hash.setdefault(value, []).append((position, f)) + index = self._entry_index[(key, kind)] = (by_name, by_hash) + return index + def _matching(self, key: str, kind: str, entry: dict) -> list[dict]: """The profile entries of `key` that read the platform entry. @@ -434,19 +453,28 @@ class EntryCategories: if n } declared = _declared_hashes(entry) - by_hash: list[dict] = [] - by_name: list[dict] = [] - for f in self._profiles[key].get("files") or []: - if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA): - continue - cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)} - if declared and _accepts(self._accepted_by(f), declared): - by_hash.append(cited) - continue - own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n} - if own & names and not (declared and _contradicts(f, declared)): - by_name.append({**cited, "by": "name"}) - return by_hash or by_name + by_name_index, by_hash_index = self._index_of(key, kind) + + def cited(f: dict) -> dict: + return {k: f[k] for k in ("name", "source_ref") if f.get(k)} + + found: dict[int, dict] = {} + for values in declared.values(): + for value in values: + for position, f in by_hash_index.get(value, ()): + found[position] = f + if _CUT_MD5 <= len(value) < _FULL_MD5: + for accepted, hits in by_hash_index.items(): + if accepted.startswith(value): + found.update(hits) + if found: + return [cited(found[p]) for p in sorted(found)] + named = {position: f for n in names for position, f in by_name_index.get(n, ())} + return [ + {**cited(named[p]), "by": "name"} + for p in sorted(named) + if not (declared and _contradicts(named[p], declared)) + ] def explain(self, entry: dict, destination: str, system: str) -> dict: """The category of an entry and what establishes it.""" diff --git a/scripts/exporter/recalbox_exporter.py b/scripts/exporter/recalbox_exporter.py index 71704304a..bad223329 100644 --- a/scripts/exporter/recalbox_exporter.py +++ b/scripts/exporter/recalbox_exporter.py @@ -245,6 +245,13 @@ class Exporter(BaseExporter): return f" " + def written_list(self, fe: NativeFile, field_name: str) -> list[str] | None: + """md5 is the comma list Bios.cpp splits and compares one by one, so + a list the truth extended is a change the report names.""" + if field_name == "md5" and len(fe.hashes("md5")) > 1: + return fe.hashes("md5") + return None + def states(self, fe: NativeFile, field_name: str) -> bool: """An element it cannot rewrite is written back as Recalbox wrote it. diff --git a/scripts/exporter/retrodeck_exporter.py b/scripts/exporter/retrodeck_exporter.py index 346c50521..b742b796c 100644 --- a/scripts/exporter/retrodeck_exporter.py +++ b/scripts/exporter/retrodeck_exporter.py @@ -96,6 +96,13 @@ class Exporter(BaseExporter): return fe.hash(field_name) return super().written(fe, field_name) + def written_list(self, fe: NativeFile, field_name: str) -> list[str] | None: + """Several md5 values are a JSON array in the manifest, as _entry + writes them: a list the truth extended is a change the report names.""" + if field_name == "md5" and len(fe.hashes("md5")) > 1: + return fe.hashes("md5") + return None + def states(self, fe: NativeFile, field_name: str) -> bool: if id(fe) in getattr(self, "_appended", ()): return False diff --git a/scripts/fileless_audit.py b/scripts/fileless_audit.py index 5baf27206..deea6b65b 100644 --- a/scripts/fileless_audit.py +++ b/scripts/fileless_audit.py @@ -128,7 +128,7 @@ def main() -> None: if unknown: raise SystemExit( "ERROR: unknown profile " + ", ".join(unknown) + ". " - + suggest_names(unknown, known, "verify.py --list-emulators") + + suggest_names(unknown, known, "python scripts/verify.py --list-emulators") ) names = args.emulators if args.all: diff --git a/scripts/generate_event_schema.py b/scripts/generate_event_schema.py index 6c2a309dc..ba8c9ce49 100644 --- a/scripts/generate_event_schema.py +++ b/scripts/generate_event_schema.py @@ -13,6 +13,8 @@ import json import sys from pathlib import Path +from artifacts import write_text_atomic + ROOT = Path(__file__).resolve().parent.parent OUTPUT = ROOT / "schemas" / "install-events.schema.json" @@ -105,7 +107,7 @@ def main(argv: list[str] | None = None) -> int: print(f"{args.output} differs from EVENT_KEYS: run " "python scripts/generate_event_schema.py", file=sys.stderr) return 1 - args.output.write_text(text, encoding="utf-8") + write_text_atomic(str(args.output), text) print(f"Wrote {args.output}") return 0 diff --git a/scripts/generate_games_index.py b/scripts/generate_games_index.py index e61db4278..de991a60a 100644 --- a/scripts/generate_games_index.py +++ b/scripts/generate_games_index.py @@ -59,6 +59,7 @@ from romfolders import ( # noqa: E402 folder_names, load_setups, ) +from artifacts import write_text_atomic # noqa: E402 from safeparse import yaml_load # noqa: E402 import install # noqa: E402 @@ -609,16 +610,60 @@ def print_coverage(coverage: dict, files: dict[Path, str]) -> None: print(f"{len(files)} files, {raw:,} bytes, {packed:,} bytes gzip") +def _sample(items: list[str], limit: int = 3) -> str: + shown = ", ".join(items[:limit]) + return f"{shown}, +{len(items) - limit} more" if len(items) > limit else shown + + +def describe_change(old: str | None, new: str) -> list[str]: + """What differs between a published document and the one render() makes. + + The documents are single-line JSON of up to 550 KB: STALE alone left a + diff by hand to find that a manifest gained one file.""" + if old is None: + return ["not written yet"] + try: + before, after = json.loads(old), json.loads(new) + except ValueError: + return ["the published file is not valid JSON"] + if not (isinstance(before, dict) and isinstance(after, dict)): + return ["document changed"] + parts = [] + for key in sorted(set(before) | set(after)): + a, b = before.get(key), after.get(key) + if a == b: + continue + if isinstance(a, list) and isinstance(b, list): + olds = {json.dumps(x, sort_keys=True) for x in a} + news = {json.dumps(x, sort_keys=True) for x in b} + added = sorted(news - olds) + removed = sorted(olds - news) + change = [] + if added: + change.append(f"{len(added)} added ({_sample(added)})") + if removed: + change.append(f"{len(removed)} removed ({_sample(removed)})") + parts.append(f"{key}: {', '.join(change) or 'reordered'}") + elif isinstance(a, dict) and isinstance(b, dict): + changed = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k)) + parts.append(f"{key}: {len(changed)} changed ({_sample(changed)})") + else: + parts.append(f"{key} changed" if key in before and key in after + else f"{key} {'added' if key in after else 'removed'}") + return parts or ["formatting changed"] + + def main() -> int: parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0]) parser.add_argument("--output-dir", type=Path, default=OUTPUT, help=f"where the index is written (default {OUTPUT})") - parser.add_argument("--check", action="store_true", help="exit 1 when a file would change") + parser.add_argument("--check", action="store_true", + help="exit 1 when a file would change, naming what differs (implies --quiet)") parser.add_argument("--quiet", action="store_true", help="print only failures") args = parser.parse_args() files, coverage = render(args.output_dir) - if not args.quiet: + if not (args.quiet or args.check): print_coverage(coverage, files) failures = floor_failures(coverage) for failure in failures: @@ -628,14 +673,22 @@ def main() -> int: ) if args.output_dir.is_dir() else [] changed = [p for p, t in files.items() if not p.exists() or p.read_text(encoding="utf-8") != t] if args.check: - for path in changed + stale: + for path in changed: print(f"STALE {path}", file=sys.stderr) + current = path.read_text(encoding="utf-8") if path.exists() else None + for line in describe_change(current, files[path]): + print(f" {line}", file=sys.stderr) + for path in stale: + print(f"STALE {path}", file=sys.stderr) + print(" no longer generated", file=sys.stderr) + if changed or stale: + print("Run python scripts/generate_games_index.py to write them.", file=sys.stderr) return 1 if changed or stale or failures else 0 for path in stale: path.unlink() for path in changed: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(files[path], encoding="utf-8") + write_text_atomic(str(path), files[path]) print(f"{len(changed)} written, {len(stale)} removed") return 1 if failures else 0 diff --git a/scripts/generate_installer_doc.py b/scripts/generate_installer_doc.py index d0cfce9bb..f19dc77c0 100644 --- a/scripts/generate_installer_doc.py +++ b/scripts/generate_installer_doc.py @@ -45,6 +45,8 @@ import zipfile from pathlib import Path, PurePosixPath from typing import Callable +from artifacts import write_text_atomic + ROOT = Path(__file__).resolve().parent.parent PAGE = ROOT / "wiki" / "installer.md" INSTALLER = ROOT / "install.py" @@ -647,7 +649,7 @@ def main(argv: list[str] | None = None) -> int: print(f"{args.page} differs from install.py and install/*.json: run " "python scripts/generate_installer_doc.py", file=sys.stderr) return 1 - args.page.write_text(text, encoding="utf-8") + write_text_atomic(str(args.page), text) print(f"Wrote {args.page}") return 0 diff --git a/scripts/generate_launcher_index.py b/scripts/generate_launcher_index.py index 7da8c7103..6592d26e4 100644 --- a/scripts/generate_launcher_index.py +++ b/scripts/generate_launcher_index.py @@ -22,6 +22,7 @@ import json import sys from pathlib import Path +from artifacts import write_text_atomic from build_port import ( BATOCERA_ZIP, MUOS_APP, @@ -121,7 +122,7 @@ def main(argv: list[str] | None = None) -> int: "python scripts/generate_launcher_index.py", file=sys.stderr) return 1 args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(text, encoding="utf-8") + write_text_atomic(str(args.output), text) print(f"Wrote {args.output}") return 0 diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 2a69cda7a..b41606dba 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -98,6 +98,7 @@ from validation import ( validated_choice, agnostic_substitute, _build_validation_index, + _parse_validation, deliverable, with_archive_members, extra_alternatives, @@ -623,6 +624,10 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool: siblings = extra_alternatives(fe, profiles) if len(siblings) < 2: return False + # A sibling the code takes without a check (Raze's "BLOOD: Unknown + # Version") accepts whatever copy fills the slot. + if any(not _parse_validation(entry.get("validation")) for entry in siblings): + return True actual = compute_hashes(local_path, frozenset({"sha1", "md5", "crc32"})) for entry in siblings: for algorithm in ("sha1", "md5", "crc32"): @@ -2725,7 +2730,8 @@ def _run_verify_packs(args: argparse.Namespace) -> None: if args.platform: print( f" {platform_name}: FAIL (no pack in {args.output_dir}; build it with " - f"generate_pack.py --platform {platform_name} --output-dir {args.output_dir})" + f"python scripts/generate_pack.py --platform {platform_name} " + f"--output-dir {args.output_dir})" ) all_ok = False failing.append(f"{platform_name} (no pack)") @@ -5077,14 +5083,22 @@ def _pack_archives(output_dir: str) -> list[str]: A part of a split pack holds a share of a platform by design and was checked against the whole when it was cut: judged as a pack it would fail conformance, and injecting a manifest would change published bytes. + Every name the builder gives holds `_BIOS_Pack`, then the system tag of + a --system platform pack (`_BIOS_Pack_Playstation.zip`): build_port.py + writes its launcher archives (RetroBIOS.zip) to the same dist/, and + taking every ZIP rewrote them with a manifest after build_port had + printed their hashes. """ return sorted( name for name in os.listdir(output_dir) - if name.endswith(".zip") and not split_pack.is_part(name) + if _PACK_NAME.search(name) and not split_pack.is_part(name) ) +_PACK_NAME = re.compile(r"_BIOS_Pack(?:_[^/]+)?\.zip$") + + def verify_and_finalize_packs( output_dir: str, db: dict, diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 11d8c3f24..e4300b034 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -22,7 +22,7 @@ Steps, numbered as the run prints them: 4c --manifest-targets per-target core lists 4d bundled_only.py --strict copies only an emulator install provides 4e generate_games_index.py install/games/: what the games-on-card mode reads - 5 consistency check verify == pack: counts, paths, every emulator pack + 5 consistency check verify == pack: counts, paths, emulator and system packs 6 --verify-packs pack integrity, extraction and hashes 7 generate_readme.py README.md and CONTRIBUTING.md generate_installer_doc.py run samples and tables of wiki/installer.md @@ -518,7 +518,7 @@ def check_emulator_packs( emulators_dir: str = "emulators", platforms_dir: str = "platforms", db_path: str = "database.json", ) -> bool: - """Build every emulator pack in memory and compare it with verify --emulator. + """Build every emulator and system pack in memory and compare it with verify. Each profile in each build it has (libretro, standalone): the paths the pack fills with the profile's files against the ones verify credits. No @@ -528,9 +528,11 @@ def check_emulator_packs( import verify # noqa: PLC0415 from common import ( # noqa: PLC0415 ProfileSelectionError, + _norm_system_id, build_zip_contents_index, load_database, load_emulator_profiles, + profiles_for_systems, select_emulator_profiles, ) @@ -541,6 +543,8 @@ def check_emulator_packs( all_ok = True compared = 0 for name in sorted(profiles): + if profiles[name].get("type") == "alias": + continue # selects the profile it names, compared under that name for standalone in (False, True): try: select_emulator_profiles([name], profiles, standalone) @@ -564,7 +568,45 @@ def check_emulator_packs( print("\n".join(lines)) all_ok = False print(f" {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}") - return all_ok + # A system pack builds every profile of the system together: a placement + # rule that only goes wrong when two profiles share a path (the bluemsx + # tree beside msx-emu's files) never meets in a one-profile pack. + systems = sorted({ + _norm_system_id(str(sid)) + for profile in profiles.values() + if profile.get("type") not in ("launcher", "alias", "test") + for sid in profile.get("systems") or [] + }) + systems_ok = True + compared = 0 + for sid in systems: + for standalone in (False, True): + matching = profiles_for_systems(profiles, [sid], standalone) + if len(matching) < 2: + continue + record = generate_pack.PackRecord() + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + generate_pack.generate_emulator_pack( + matching, emulators_dir, db, "bios", "", standalone, zip_contents, + offline=True, platforms_dir=platforms_dir, record=record, + system_ids=[sid], + ) + report = verify.verify_emulator( + matching, emulators_dir, db, standalone, platforms_dir=platforms_dir, + system_ids=[sid], + ) + compared += 1 + label = f"--system {sid}{' --standalone' if standalone else ''}" + lines = _set_difference_lines( + label, set(report["pack_destinations"]), set(record.files) + ) + if lines: + print("\n".join(lines)) + systems_ok = False + print(f" {compared} system packs compared, " + f"{'all equal' if systems_ok else 'differences above'}") + return all_ok and systems_ok class _Skipped: diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 3245ab3b0..7fde29014 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -1257,6 +1257,14 @@ def _follow_fork_base( base = upstream.fork_base( view.repo, view.head, source.repo, source.head, cache_dir, offline ) + if base and base != view.pin: + # A pin newer than the merge base names code the port does not + # carry: the upstream is a build of its own, the Qt melonDS or the + # EasyRPG Player a standalone + libretro profile documents, and + # its HEAD stays the reference. melonds read 825 commits back. + moved = upstream.ancestry(view.repo, base, view.pin, cache_dir, offline) + if moved and moved[0] > 0 and moved[1] == 0: + base = None followed.append( replace(view, head=base, origin=f"{view.origin}, fork base of {source.repo.slug}") if base else view @@ -2627,6 +2635,7 @@ def tree_diff(result: CompareResult, ref_dirs: set[str]) -> list[str]: DRIFT_WEIGHTS = {"GONE": 40, "CHANGED": 30, "AMBIGUOUS": 15, "SHIFTED": 1} +PIN_WEIGHT = 50 def drift_score( @@ -2644,6 +2653,10 @@ def drift_score( if version is not None and not version.tag_matches_declared: if version.latest_release and version.latest_release != version.declared: score += 10 + # A pin off its branch or a repository answering under another name is + # set again before any ref can be read, so each outranks a GONE ref: + # pcem ranked last at 0 with an orphaned upstream_commit. + score += PIN_WEIGHT * (len(report.orphaned) + len(report.moved_repos)) score += min(commits, 20) // 4 return score @@ -4154,16 +4167,22 @@ def _refuse_dirty_tree(emulators_dir: str, names: list[str]) -> None: print(f"{e}. Pass --force to write anyway.", file=sys.stderr) raise SystemExit(1) from e if dirty: + # Another session may own the change, so the refusal names the files: + # --force writes over them. + one = len(dirty) == 1 print( - f"{emulators_dir} carries uncommitted changes to the selected " - "profiles. Commit them first or pass --force.", + f"{', '.join(dirty)} {'has' if one else 'have'} uncommitted changes: " + f"commit {'it' if one else 'them'} or pass --force, which writes over " + f"{'it' if one else 'them'}.", file=sys.stderr, ) raise SystemExit(1) -def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) -> bool: - """True when the profile directory carries uncommitted changes. +def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) -> list[str]: + """The files of the profile directory that carry uncommitted changes. + + Paths read as the status prints them, from the repository root. ``names`` narrows the question to those profiles. Git is asked from inside the directory, so a profile tree that lives in another @@ -4181,7 +4200,11 @@ def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) - raise RuntimeError( f"cannot read the git status of {emulators_dir}: {result.stderr.strip()}" ) - return bool(result.stdout.strip()) + return sorted( + line[3:].split(" -> ")[-1].strip('"') + for line in result.stdout.splitlines() + if line.strip() + ) def build_parser() -> argparse.ArgumentParser: @@ -4414,7 +4437,7 @@ def select_profiles(profiles: dict, args: argparse.Namespace) -> dict: ) raise SystemExit( f"ERROR: unknown profile {', '.join(unknown)}. " - + suggest_names(unknown, listed, "verify.py --list-emulators") + + suggest_names(unknown, listed, "python scripts/verify.py --list-emulators") ) return {name: profiles[name] for name in dict.fromkeys(names)} @@ -4451,7 +4474,7 @@ def select_profiles(profiles: dict, args: argparse.Namespace) -> dict: if args.system: # Suggested from the spellings the named listing prints. systems = sorted(spelling for spelling, _ in system_index(profiles).values()) - hint = " " + suggest_names([args.system], systems, "verify.py --list-systems") + hint = " " + suggest_names([args.system], systems, "python scripts/verify.py --list-systems") elif args.type_: kinds = sorted({str(p.get("type")) for p in profiles.values() if p.get("type")}) hint = f" Types: {', '.join(kinds)}." @@ -4604,8 +4627,14 @@ def _apply_writes( # The guard above read the prose under the pins the profile held. # A bump that moves upstream_commit can leave prose the report # judged in the fork under the new upstream pin (dolphin-mmjr2, - # hatari, xenios): read it again on what was written. - left = realign_prose(path, args.cache_dir, args.offline, dry_run=True) + # hatari, xenios): read it again on what was written. It reads + # the profile's own history, so it runs on the real path, and a + # read that fails (offline, rate limit) puts the profile back. + try: + left = realign_prose(path, args.cache_dir, args.offline, dry_run=True) + except BaseException: + path.write_bytes(original) + raise if moved and left: path.write_bytes(original) print( diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py index 44c7ca5ba..77d217143 100644 --- a/scripts/provenance_report.py +++ b/scripts/provenance_report.py @@ -66,7 +66,8 @@ def profile_needs(profiles: dict, db: dict) -> dict: Keyed by every hash a profile accepts, and, for an entry that declares a size and no hash, by its names casefolded. An entry with neither - cannot be told apart from any same-named dump and is left out. + cannot be told apart from any same-named dump and is left out, as is a + file the emulator installs itself (`bundled:`) on every platform. """ files = db.get("files", {}) indexes = db.get("indexes", {}) @@ -92,6 +93,10 @@ def profile_needs(profiles: dict, db: dict) -> dict: name = str(entry.get("name", "")) if not name or name.endswith("/"): continue + # Installed with the emulator, so not an acquisition target, + # unless a platform package leaves it out. + if entry.get("bundled") and not entry.get("not_bundled_on"): + continue need = {"emulator": emulator, "name": name, "required": bool(entry.get("required"))} hashes = {(field, v) for field in HASH_FIELDS for v in _values(entry, field)} diff --git a/scripts/validation.py b/scripts/validation.py index 300e36d05..544451f90 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -8,6 +8,7 @@ truth data for reporting. from __future__ import annotations import os +import re import zipfile from collections.abc import Callable, Iterator, Mapping, Sequence @@ -161,6 +162,11 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: Raises ValueError if two profiles declare conflicting values. """ index: dict[str, dict] = {} + # A slot where one version is taken without any check (Raze's "BLOOD: + # Unknown Version", which holds the shareware RFF) accepts every copy for + # that core: its validated siblings are alternatives, not requirements. + unchecked: set[tuple[str, str, str]] = set() + checked: set[tuple[str, str, str]] = set() for emu_name, profile in profiles.items(): if profile.get("type") in ("launcher", "alias"): continue @@ -169,8 +175,11 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: if not fname: continue checks = _parse_validation(f.get("validation")) + slot = (fname, emu_name, f.get("path") or fname) if not checks: + unchecked.add(slot) continue + checked.add(slot) if fname not in index: index[fname] = { "checks": set(), @@ -276,6 +285,8 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: existing["source_ref"] = pe_entry["source_ref"] else: pe[emu_name] = pe_entry + for fname, emu_name, _path in sorted(checked & unchecked): + index[fname]["rules"].append(_entry_rule(emu_name, {}, set())) # Convert sets to sorted tuples/lists for determinism for v in index.values(): v["checks"] = sorted(v["checks"]) @@ -324,6 +335,23 @@ def _emulators_for_check( return sorted(result) +_REASON = re.compile(r"^(\w+ mismatch): got (\S+), accepted \[(.*)\]$") + + +def _merged_reason(reasons: list[str]) -> str: + """One reason for every entry of a core that turned the copy away. + + Raze lists three BLOOD.RFF sizes and the message named the first one + alone. Reasons of one kind are merged into a single accepted list. + """ + parsed = [_REASON.match(reason) for reason in reasons] + if all(parsed) and len({(m.group(1), m.group(2)) for m in parsed}) == 1: + accepted = sorted({v for m in parsed for v in m.group(3).split(",") if v}, + key=lambda v: (len(v), v)) + return f"{parsed[0].group(1)}: got {parsed[0].group(2)}, accepted [{','.join(accepted)}]" + return reasons[0] + + def check_file_validation( local_path: str, filename: str, @@ -363,7 +391,7 @@ def check_file_validation( for rule in rules ] if all(reasons): - failures[emu] = reasons[0] + failures[emu] = _merged_reason(reasons) if not failures: return None first = sorted(failures)[0] diff --git a/scripts/verify.py b/scripts/verify.py index 67aef9f42..f9e47d8c6 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1827,7 +1827,11 @@ def _omission_reasons(entries: list[dict]) -> str: def _print_undeclared_section(result: dict, verbose: bool) -> None: """Print cross-reference section for undeclared files used by cores.""" undeclared = result.get("undeclared_files", []) - if not undeclared: + # The pack's core copies can come from no gap entry: recalbox + # --system nec-pc-engine packs six root syscard copies for geargrafx + # with an empty gap report, and returning here hid them. + packed = result.get("packed_extras") + if not undeclared and not (packed and any(packed.values())): return game_data, bios_files, unplaced = _split_undeclared(undeclared) @@ -1850,7 +1854,6 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: # The builder's count, one file per pack path, when the platform pass # replayed it. Counting entries said 1130 for RetroArch, 48 more than the # pack holds: rom.key alone was claimed by three Amiga cores. - packed = result.get("packed_extras") core_in_pack = ( packed["bios"] if packed is not None else len(req_in_repo) + len(opt_in_repo) ) @@ -1880,7 +1883,7 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: print(f" Core files held but not packed: {len(unplaced)} " f"({_omission_reasons(unplaced)})") - if game_data: + if game_data or (packed and packed.get("game_data")): gd_missing = [u for u in game_data if not u["in_repo"]] gd_packed = ( packed["game_data"] if packed is not None @@ -2113,6 +2116,30 @@ def verify_emulator( return placed[dest.lower()] = (path, owner, dest) _register_path(dest, placed_files, placed_parents) + + def occupy_trees(emu_name: str, profile: dict) -> None: + """The profile's data trees follow its files in the pack, which + writes them even when the system filter leaves it no file: + bluemsx's Machines/ tree holds msx-emu's coleco.rom path in a + --system colecovision pack.""" + for dd in deliverable(profile.get("data_directories", []), standalone): + cache = (data_registry or {}).get(dd.get("ref", ""), {}).get("local_cache", "") + if not cache or not os.path.isdir(cache): + continue + dd_dest = dd.get("destination", "") + structure = profile.get("pack_structure") + prefix = ( + structure.get("standalone" if standalone else "libretro", "") + if structure else "" + ) + if prefix: + dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix + for root, _dirs, names in os.walk(cache): + for fname in names: + src = os.path.join(root, fname) + rel = os.path.relpath(src, cache).replace(os.sep, "/") + occupy(f"{dd_dest}/{rel}" if dd_dest else rel, src, emu_name) + for emu_name, profile in sorted(selected): files = expand_agnostic_entries( expand_directory_entries( @@ -2159,6 +2186,7 @@ def verify_emulator( "ground_truth": [], } ) + occupy_trees(emu_name, profile) continue # Verify archives as units (e.g., neogeo.zip, aes.zip) @@ -2421,24 +2449,7 @@ def verify_emulator( ) > _SEVERITY_ORDER.get(prev_sev, 0): file_severity[dest] = sev - # The profile's data trees follow its files in the pack. - for dd in deliverable(profile.get("data_directories", []), standalone): - cache = (data_registry or {}).get(dd.get("ref", ""), {}).get("local_cache", "") - if not cache or not os.path.isdir(cache): - continue - dd_dest = dd.get("destination", "") - structure = profile.get("pack_structure") - prefix = ( - structure.get("standalone" if standalone else "libretro", "") - if structure else "" - ) - if prefix: - dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix - for root, _dirs, names in os.walk(cache): - for fname in names: - src = os.path.join(root, fname) - rel = os.path.relpath(src, cache).replace(os.sep, "/") - occupy(f"{dd_dest}/{rel}" if dd_dest else rel, src, emu_name) + occupy_trees(emu_name, profile) counts = { Severity.OK: 0, diff --git a/scripts/ziptools.py b/scripts/ziptools.py index cb82d41b9..e8ad8bdfb 100644 --- a/scripts/ziptools.py +++ b/scripts/ziptools.py @@ -14,7 +14,7 @@ import sys import tempfile import zipfile -from artifacts import set_plain_mode +from artifacts import process_umask, set_plain_mode from hashing import md5sum @@ -251,9 +251,7 @@ def safe_extract_zip( """ dest = os.path.realpath(dest_dir) os.makedirs(dest, exist_ok=True) - # Reading the umask sets it, so it is put back at once. - umask = os.umask(0) - os.umask(umask) + umask = process_umask() with zipfile.ZipFile(zip_path, "r") as zf: members = zf.infolist() if len(members) > max_members: diff --git a/tests/test_atomic_artifacts.py b/tests/test_atomic_artifacts.py index e106975b8..ad3868035 100644 --- a/tests/test_atomic_artifacts.py +++ b/tests/test_atomic_artifacts.py @@ -83,6 +83,27 @@ class EveryWriterGoesThroughIt(unittest.TestCase): ] self.assertEqual(offenders, []) + def test_generators_publish_whole_files(self): + """install/games/ (12 MB, read by install.py --for-games), the event + schema, the launcher index and wiki/installer.md were truncated at + their final name: an interrupted run left a committed-looking file + holding part of a JSON document. The installer doc's sandbox writes + throwaway fixtures under tmp/ and is not published.""" + import ast + + sandbox = {"write_small_list", "Sandbox"} + offenders = [] + for path in sorted((REPO_ROOT / "scripts").glob("generate_*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + for top in tree.body: + if path.name == "generate_installer_doc.py" and getattr(top, "name", "") in sandbox: + continue + for node in ast.walk(top): + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr in ("write_text", "write_bytes")): + offenders.append(f"{path.name}:{node.lineno}") + self.assertEqual(offenders, []) + def test_the_text_is_utf8_whatever_the_locale(self): import tempfile @@ -154,6 +175,38 @@ class ScratchFilesGetThePlainMode(unittest.TestCase): artifacts.write_text_atomic(str(self.dir / "a.json"), "{}") self.assertEqual(self._mode(self.dir / "a.json"), 0o664) + def test_threads_reading_the_mode_leave_the_umask_alone(self): + """mame_devices fetches on eight threads and every cache write reads + the umask: unguarded, two readers interleaved and left it at 0.""" + import threading # noqa: PLC0415 + + interval = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + self.addCleanup(sys.setswitchinterval, interval) + modes: set[int] = set() + + def read() -> None: + seen = {artifacts.plain_file_mode() for _ in range(20000)} + modes.update(seen) + + threads = [threading.Thread(target=read) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + current = os.umask(0o022) + self.assertEqual(current, 0o022) + self.assertEqual(modes, {0o644}) + + def test_no_script_reads_the_umask_unguarded(self): + offenders = [ + f"{path.name}:{number}" + for path in sorted((REPO_ROOT / "scripts").rglob("*.py")) + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1) + if "os.umask(" in line and path.name != "artifacts.py" + ] + self.assertEqual(offenders, []) + def test_packs_and_their_manifest(self): import zipfile # noqa: PLC0415 diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index e6f95068d..fbb7afeff 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -2725,6 +2725,34 @@ class DeclaredDependenciesMatchTheDocumentedWorkflows(unittest.TestCase): ) +class TheProjectInstallsWithPip(unittest.TestCase): + """pyproject declared a dev extra that pip could not install: without a + [build-system] and an explicit module list, setuptools' flat-layout + discovery found scripts/ and tests/ and refused to build.""" + + def test_setuptools_discovers_nothing_to_package(self): + import importlib.util # noqa: PLC0415 + + if importlib.util.find_spec("setuptools") is None: + self.skipTest("setuptools is not installed") + program = ( + "from setuptools.dist import Distribution\n" + "from setuptools.config.pyprojecttoml import apply_configuration\n" + "dist = apply_configuration(Distribution(), 'pyproject.toml')\n" + "dist.set_defaults()\n" + "print(sorted(dist.packages or []), sorted(dist.py_modules or []))\n" + ) + done = subprocess.run([sys.executable, "-c", program], cwd=ROOT, + capture_output=True, text=True, check=False) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + self.assertEqual(done.stdout.strip(), "[] []") + import tomllib # noqa: PLC0415 + + with (ROOT / "pyproject.toml").open("rb") as handle: + build = tomllib.load(handle)["build-system"] + self.assertEqual(build["build-backend"], "setuptools.build_meta") + + class ACheckThatCannotAnswerDoesNotPass(unittest.TestCase): """Exiting zero says the question was answered and the answer was yes. @@ -4196,6 +4224,10 @@ class AutoFetchAsksVerifyWhatIsMissing(unittest.TestCase): "system": "diablo", "emulator": "DevilutionX"}, {"name": "held.bin", "in_repo": True, "required": True}, {"name": "extra.bin", "in_repo": False, "required": False}, + # zesarux installs 128s.rom with the emulator: not a target, + # and a copy placed in bios/ fails pipeline step 4d. + {"name": "128s.rom", "in_repo": False, "required": True, + "bundled": True, "emulator": "ZEsarUX"}, ], } with mock.patch.object(verify_mod, "verify_platform", lambda c, d: result): @@ -5582,12 +5614,17 @@ class ANamedPackThatIsAbsentFails(unittest.TestCase): -class AnAliasNamesItsProfile(unittest.TestCase): - """"flycast_gles2 is an alias of flycast -use --emulator flycast": the - dash read as an option, and an alias without alias_of advised - "--emulator ?".""" +class AnAliasSelectsItsProfile(unittest.TestCase): + """`--emulator mednafen_psx` refused with "use --emulator beetle_psx", + a step the tool could take itself: the name a user reads in RetroArch + now selects the profile it is an alias of, and says so on stderr. An + alias without alias_of is found through the profile whose cores: names + it.""" + + def test_the_alias_selects_its_profile(self): + import contextlib as _contextlib + import io as _io - def test_the_message(self): from scripts.common import ProfileSelectionError, select_emulator_profiles profiles = { @@ -5595,12 +5632,28 @@ class AnAliasNamesItsProfile(unittest.TestCase): "flycast_gles2": {"type": "alias", "alias_of": "flycast"}, "orphan": {"type": "alias"}, "core": {"type": "libretro", "cores": ["orphan"]}, + "lost": {"type": "alias"}, } - for name, expected in (("flycast_gles2", "alias of flycast: use --emulator flycast"), - ("orphan", "alias of core: use --emulator core")): - with self.subTest(name=name), self.assertRaises(ProfileSelectionError) as caught: - select_emulator_profiles([name], profiles) - self.assertIn(expected, str(caught.exception)) + for name, expected in (("flycast_gles2", "flycast"), ("orphan", "core")): + err = _io.StringIO() + with self.subTest(name=name), _contextlib.redirect_stderr(err): + selected = select_emulator_profiles([name], profiles) + self.assertEqual([key for key, _ in selected], [expected]) + self.assertIn(f"{name} is {expected}", err.getvalue()) + with _contextlib.redirect_stderr(_io.StringIO()): + both = select_emulator_profiles(["flycast", "flycast_gles2"], profiles) + self.assertEqual([key for key, _ in both], ["flycast"]) + with self.assertRaises(ProfileSelectionError): + select_emulator_profiles(["lost"], profiles) + + def test_verify_runs_the_alias(self): + result = subprocess.run( + [sys.executable, "scripts/verify.py", "--emulator", "mednafen_psx"], + cwd=ROOT, capture_output=True, text=True, timeout=300, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("beetle_psx", result.stdout) + self.assertIn("mednafen_psx is beetle_psx", result.stderr) diff --git a/tests/test_autorun.py b/tests/test_autorun.py index aa32b1d05..c8c04e0ad 100644 --- a/tests/test_autorun.py +++ b/tests/test_autorun.py @@ -955,6 +955,23 @@ class SubjectsFollowTheRepositoryRules(unittest.TestCase): self.assertTrue(ar.check_subject(subject)) self.assertEqual(ar.check_subject("chore: tidy the cache"), []) + def test_review_numbers_are_refused(self): + """47 pushed subjects pointed at a numbered private review, which no + clone can follow. The subject says what the change does.""" + import commit_rules # noqa: PLC0415 + + for subject in ("chore: audit 24 and 17 cases, host kept out", + "fix: polish the screens audit 11 named", + "chore: merge audit batch, resolver and profiles", + "fix: settle the review 3 notes"): + with self.subTest(subject=subject): + self.assertTrue(commit_rules.problems(subject)) + for subject in ("fix: scroll long messages in the installer", + "feat: read regions in region_audit.py", + "chore: merge resolver and profile fixes"): + with self.subTest(subject=subject): + self.assertEqual(commit_rules.problems(subject), []) + def test_the_rules_run_as_a_hook(self): with tempfile.TemporaryDirectory() as tmp: message = Path(tmp) / "MSG" diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 409e80248..8b42cafaa 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -538,7 +538,7 @@ class ProfileFieldsRequired(unittest.TestCase): section = _section(_page("profiling.md"), "Profile fields") marked = { row[0].strip("`") for row in _table_rows(section) - if row[1].startswith(("yes", "for `type")) + if row[1].startswith(("yes", "for `type", "with `")) } self.assertEqual(marked, required) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 4d0b203cd..b715cbb7f 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -17,7 +17,9 @@ Covers: from __future__ import annotations +import contextlib import hashlib +import io import os import shutil import sys @@ -1444,10 +1446,13 @@ class TestE2E(unittest.TestCase): with self.assertRaises(SystemExit): verify_emulator(["test_game"], self.emulators_dir, self.db, standalone=True) - def test_93_verify_emulator_alias_rejected(self): - """Alias profile produces error with redirect message.""" - with self.assertRaises(SystemExit): - verify_emulator(["test_alias"], self.emulators_dir, self.db) + def test_93_verify_emulator_alias_selects_its_profile(self): + """An alias verifies the profile it is an alias of.""" + with contextlib.redirect_stderr(io.StringIO()): + via_alias = verify_emulator(["test_alias"], self.emulators_dir, self.db) + direct = verify_emulator(["test_emu"], self.emulators_dir, self.db) + self.assertEqual(via_alias["emulators"], direct["emulators"]) + self.assertEqual(via_alias["details"], direct["details"]) def test_94_verify_emulator_launcher_rejected(self): """Launcher profile produces error.""" diff --git a/tests/test_entry_categories.py b/tests/test_entry_categories.py index d3c68861c..0f971c5d9 100644 --- a/tests/test_entry_categories.py +++ b/tests/test_entry_categories.py @@ -220,5 +220,49 @@ class IndexedProof(unittest.TestCase): self.assertEqual(index.proofs(entry, destination), expected) +class AProfileIsIndexedOncePerKind(unittest.TestCase): + """_matching walked the reader's whole profile for each platform entry: + 2,135 scans of the MAME profile took 2 s of verify --all.""" + + FILES = [ + {"name": "zero.bin", "md5": "0" * 32, "source_ref": "a.c:1"}, + {"name": "one.bin", "md5": "1" * 32, "source_ref": "a.c:2"}, + {"name": "one.bin", "md5": "2" * 32, "source_ref": "a.c:3"}, + {"name": "two.bin", "aliases": ["TWO.ROM"], "source_ref": "a.c:4"}, + {"name": "data.pak", "category": "game_data", "source_ref": "a.c:5"}, + ] + + def setUp(self): + profiles = {"core": {"emulator": "Core", "type": "libretro", "cores": ["core"], + "systems": ["s"], "files": self.FILES}} + self.cats = EntryCategories(_config(["core"], []), profiles) + + def test_the_index_answers_as_the_scan_did(self): + cases = [ + ({"name": "one.bin", "md5": "2" * 32}, BIOS, [{"name": "one.bin", "source_ref": "a.c:3"}]), + # A cut md5 (Batocera's 29 characters) matches by prefix. + ({"name": "x", "md5": "1" * 29}, BIOS, [{"name": "one.bin", "source_ref": "a.c:2"}]), + ({"name": "one.bin"}, BIOS, [ + {"name": "one.bin", "source_ref": "a.c:2", "by": "name"}, + {"name": "one.bin", "source_ref": "a.c:3", "by": "name"}, + ]), + ({"name": "one.bin", "md5": "9" * 32}, BIOS, []), + ({"name": "two.rom"}, BIOS, [{"name": "two.bin", "source_ref": "a.c:4", "by": "name"}]), + ({"name": "data.pak"}, BIOS, []), + ({"name": "data.pak"}, GAME_DATA, [{"name": "data.pak", "source_ref": "a.c:5", "by": "name"}]), + ] + for entry, kind, expected in cases: + with self.subTest(entry=entry, kind=kind): + self.assertEqual(self.cats._matching("core", kind, entry), expected) + + def test_the_profile_is_walked_once_per_kind(self): + from unittest import mock + + with mock.patch.object(self.cats, "_accepted_by", wraps=self.cats._accepted_by) as hashed: + for _ in range(50): + self.cats._matching("core", BIOS, {"name": "zero.bin", "md5": "0" * 32}) + self.assertEqual(hashed.call_count, 4) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py index ac46526b8..f0abfc261 100644 --- a/tests/test_export_counts.py +++ b/tests/test_export_counts.py @@ -1060,5 +1060,41 @@ class ARemovedSizeIsReported(unittest.TestCase): self.assertEqual(changes.unsized, ["SNK - NeoGeo CD/neocdz.zip"]) +class AnExtendedListIsReported(unittest.TestCase): + """The truth adds an accepted md5 the platform lacks without + contradicting it: RetroDECK wrote [A, B] where the manifest had A and + Recalbox md5="A,B" where es_bios.xml had A, and the report listed + nothing. Only Batocera named the list it writes.""" + + def _records(self, exporter): + from export_native import landed_changes # noqa: PLC0415 + + platform = {"name": "f.bin", "md5": A, "core": "libretro/mesen"} + fe = NativeFile("f.bin", "f.bin", "s", platform=platform, + truth={"md5": [A, B], "validation": ["md5"]}) + return landed_changes(exporter, {"s": NativeSystem("s", files=[fe])}).records, fe + + def test_both_formats_report_the_list_they_write(self): + for exporter in (RetroDeck(), Recalbox()): + with self.subTest(exporter=exporter.platform_name()): + records, fe = self._records(exporter) + self.assertEqual(fe.hashes("md5"), [A, B]) + self.assertEqual( + [(r["kind"], r["field"], r["old"], r["new"]) for r in records], + [("hash_corrected", "md5", A, [A, B])], + ) + + def test_an_unchanged_value_reports_nothing(self): + from export_native import landed_changes # noqa: PLC0415 + + for exporter in (RetroDeck(), Recalbox()): + fe = NativeFile("f.bin", "f.bin", "s", + platform={"name": "f.bin", "md5": A, "core": "libretro/mesen"}, + truth={"md5": A}) + with self.subTest(exporter=exporter.platform_name()): + changes = landed_changes(exporter, {"s": NativeSystem("s", files=[fe])}) + self.assertEqual(changes.records, []) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_games_index.py b/tests/test_games_index.py index f47522132..b7894b42f 100644 --- a/tests/test_games_index.py +++ b/tests/test_games_index.py @@ -176,6 +176,47 @@ class GamesIndexFixture(unittest.TestCase): self.assertTrue(any("catalog" in f for f in failures)) +class CheckSaysWhatDiffers(unittest.TestCase): + """--check printed STALE and a path after 120 lines of coverage: the + reader had to diff single-line JSON of 550 KB by hand to find that a + manifest had gained one file.""" + + def test_an_added_file_is_named(self) -> None: + old = json.dumps({"files": [["a.bin", 1, "bios"]], "version": 1}) + new = json.dumps({"files": [["a.bin", 1, "bios"], ["ide12.rom", 16384, "bios"]], + "version": 1}) + lines = gi.describe_change(old, new) + self.assertEqual(len(lines), 1) + self.assertIn("1 added", lines[0]) + self.assertIn("ide12.rom", lines[0]) + + def test_new_and_unreadable_documents(self) -> None: + self.assertEqual(gi.describe_change(None, "{}"), ["not written yet"]) + self.assertIn("not valid JSON", gi.describe_change("{", "{}")[0]) + + def test_check_names_the_command_and_skips_coverage(self) -> None: + import contextlib + import io + from unittest import mock + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + target = out / "index.json" + target.write_text(json.dumps({"files": []}), encoding="utf-8") + files = {target: json.dumps({"files": [["x.rom", 2, "bios"]]})} + coverage = {"setups": {}, "systems": {}, "arcade": []} + stdout, stderr = io.StringIO(), io.StringIO() + argv = ["generate_games_index.py", "--check", "--output-dir", tmp] + with mock.patch.object(gi, "render", return_value=(files, coverage)), \ + mock.patch.object(sys, "argv", argv), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = gi.main() + self.assertEqual(code, 1) + self.assertEqual(stdout.getvalue(), "") + self.assertIn("files: 1 added", stderr.getvalue()) + self.assertIn("python scripts/generate_games_index.py", stderr.getvalue()) + + class CommittedData(unittest.TestCase): """The data the repository publishes.""" diff --git a/tests/test_native_mode.py b/tests/test_native_mode.py index e913bb930..100412e98 100644 --- a/tests/test_native_mode.py +++ b/tests/test_native_mode.py @@ -10,6 +10,8 @@ it, and these tests make the agreement checkable instead of reviewable. from __future__ import annotations +import contextlib +import io import json import sys import unittest @@ -446,10 +448,12 @@ class OneProfileSelector(unittest.TestCase): self.assertIsNone(error) self.assertEqual([n for n, _p in selected], ["real"]) - def test_an_alias_names_the_profile_to_ask_instead(self): - _selected, error = self._select(["ghost"]) - self.assertIn("alias of real", error) - self.assertIn("--emulator real", error) + def test_an_alias_selects_the_profile_it_names(self): + with contextlib.redirect_stderr(io.StringIO()) as err: + selected, error = self._select(["ghost"]) + self.assertIsNone(error) + self.assertEqual([n for n, _p in selected], ["real"]) + self.assertIn("ghost is real", err.getvalue()) def test_a_launcher_is_refused(self): _selected, error = self._select(["starter"]) diff --git a/tests/test_pack_destinations.py b/tests/test_pack_destinations.py index d5bbd80bd..0cbfb8adb 100644 --- a/tests/test_pack_destinations.py +++ b/tests/test_pack_destinations.py @@ -189,6 +189,9 @@ class EveryEmulatorPackMatchesItsReport(unittest.TestCase): finally: os.chdir(previous) self.assertTrue(ok, out.getvalue()) + # Packs of several profiles too: a placement rule that only goes + # wrong when two profiles share a path never meets in one profile. + self.assertIn("system packs compared, all equal", out.getvalue()) if __name__ == "__main__": diff --git a/tests/test_pin_build.py b/tests/test_pin_build.py new file mode 100644 index 000000000..c4f7a4568 --- /dev/null +++ b/tests/test_pin_build.py @@ -0,0 +1,138 @@ +"""Held pins name the build file that holds them. + +pin_frozen was free prose and no tool read the build file it cites: a +platform moving mame0278 to another commit would have left the profile +holding a revision no platform builds, in silence. `pin_build` records the +file, and check_freshness reads it again where the platform now stands. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import check_freshness as cf # noqa: E402 + +RECALBOX = "https://gitlab.com/recalbox/recalbox" + + +class HeldPinsReadTheirBuildFileAgain(unittest.TestCase): + REF = {"platform": "recalbox", "repo": RECALBOX, + "revision": "10.1.1", "path": "package/libretro-mame0278/libretro-mame0278.mk", + "variable": "LIBRETRO_MAME0278_VERSION", "value": "a" * 40} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.platforms = Path(self._tmp.name) + + def _platform(self, revision: str) -> None: + (self.platforms / "recalbox.yml").write_text( + f'source: "{RECALBOX}/-/raw/{revision}/board/es_bios.xml"\n' + ) + + def _check(self, lines=None, offline=False): + profiles = {"mame0278": {"pin_frozen": "Recalbox builds it.", "pin_build": [self.REF]}, + "raine": {"pin_frozen": "RetroBat installs this release."}} + with mock.patch.object(cf.upstream, "fetch_file", return_value=lines) as fetched: + findings = cf.check_pin_builds(profiles, self.platforms, "cache", offline) + return {f.subject: f for f in findings}, fetched + + def test_the_variable_forms_the_platforms_write(self): + cases = [ + (["LIBRETRO_MAME0278_VERSION = fba5074"], "LIBRETRO_MAME0278_VERSION", "fba5074"), + (['PKG_VERSION="37cddf2" #v0.6.3'], "PKG_VERSION", "37cddf2"), + (['rp_module_repo="file $__archive_url/advancemame-1.4.tar.gz"'], "rp_module_repo", + "file $__archive_url/advancemame-1.4.tar.gz"), + (["function f() {", ' local version="20250829"', ' version="20250302"'], + "version", "20250829"), + (["OTHER_VERSION = 1"], "PKG_VERSION", None), + ] + for lines, variable, expected in cases: + with self.subTest(variable=variable): + self.assertEqual(cf.build_variable(lines, variable), expected) + + def test_the_platform_read_at_the_same_revision_needs_no_network(self): + self._platform("10.1.1") + found, fetched = self._check() + fetched.assert_not_called() + self.assertEqual(found["mame0278 (recalbox)"].status, cf.OK) + + def test_a_new_revision_building_another_commit_is_stale(self): + self._platform("10.2") + found, fetched = self._check([f"LIBRETRO_MAME0278_VERSION = {'b' * 40}"]) + self.assertEqual(fetched.call_args.args[1:3], + ("10.2", "package/libretro-mame0278/libretro-mame0278.mk")) + row = found["mame0278 (recalbox)"] + self.assertEqual(row.status, cf.STALE) + self.assertIn("now builds " + "b" * 40, row.detail) + + def test_a_new_revision_building_the_same_commit_is_ok(self): + self._platform("10.2") + found, _ = self._check([f"LIBRETRO_MAME0278_VERSION = {'a' * 40}"]) + self.assertEqual(found["mame0278 (recalbox)"].status, cf.OK) + + def test_a_build_file_gone_is_stale(self): + self._platform("10.2") + found, _ = self._check(None) + self.assertEqual(found["mame0278 (recalbox)"].status, cf.STALE) + self.assertIn("is gone", found["mame0278 (recalbox)"].detail) + + def test_offline_and_uncached_is_skipped(self): + self._platform("10.2") + profiles = {"mame0278": {"pin_frozen": "x", "pin_build": [self.REF]}} + cold = cf.upstream.NotCachedError("not cached") + with mock.patch.object(cf.upstream, "fetch_file", side_effect=cold): + (row,) = cf.check_pin_builds(profiles, self.platforms, "cache", True) + self.assertEqual(row.status, cf.SKIPPED) + + def test_a_hold_with_no_build_file_is_counted(self): + self._platform("10.1.1") + found, _ = self._check() + row = found["pin_frozen without pin_build"] + self.assertEqual(row.status, cf.SKIPPED) + self.assertEqual(row.items, ["raine"]) + + +class EveryBuildReferenceNamesThePin(unittest.TestCase): + """Reads the repository: a pin_build value is what the platform builds, + so it is the profile's source_commit or names its core_version, and the + platform's own YAML was scraped from that repository.""" + + def test_the_values_agree(self): + from common import load_emulator_profiles + + disagree = [] + for name, profile in load_emulator_profiles(str(REPO_ROOT / "emulators")).items(): + references = profile.get("pin_build") or [] + if not references: + continue + pins = profile.get("source_commit") + pins = set(pins.values()) if isinstance(pins, dict) else {pins} + versions = profile.get("core_version") + versions = ( + {str(v) for v in versions.values()} if isinstance(versions, dict) + else {str(versions)} if versions else set() + ) + + def names_it(value: str) -> bool: + token = value.split()[-1].rsplit("/", 1)[-1].removesuffix(".tar.gz") + return value in pins or any(token in v or v in value for v in versions) + + if not any(names_it(str(r["value"])) for r in references): + disagree.append(name) + for reference in references: + if not cf.pin_build_revision(reference, REPO_ROOT / "platforms"): + disagree.append(f"{name}: {reference['platform']} is not read from " + f"{reference['repo']}") + self.assertEqual(disagree, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 0fef264af..92b51c817 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -2670,6 +2670,19 @@ class TestWriteDryRun(unittest.TestCase): counts={"SHIFTED": 1}, ) + def test_a_prose_read_that_fails_leaves_the_profile_as_it_was(self): + """The bumped profile was promoted before the prose under its new pin + was read. Offline, an uncached cited file raised NotCachedError past + the restore and the profile kept a pin the guard would refuse.""" + from unittest import mock # noqa: PLC0415 + + before = self.path.read_bytes() + cold = profile_sync.upstream.NotCachedError("x.c not cached") + with mock.patch.object(profile_sync, "realign_prose", side_effect=[[], cold]), \ + self.assertRaises(profile_sync.upstream.NotCachedError): + self._run(self._args(rebase_refs=True, dry_run=False), self._shifted()) + self.assertEqual(self.path.read_bytes(), before) + def test_rebase_states_the_move_and_leaves_the_file(self): before = self.path.read_text() output = self._run(self._args(rebase_refs=True), self._shifted()) @@ -3812,6 +3825,18 @@ class TestDriftScore(unittest.TestCase): report = ProfileReport(name="a", entries=[], counts={"ANCHORED": 5}) self.assertEqual(drift_score(report, None, 0), 0) + def test_a_pin_to_set_again_outranks_a_gone_ref(self): + """pcem ranked last at 0 with "1 to review, 1 orphaned pin": its + upstream_commit had left every branch of a force-pushed repository.""" + gone = ProfileReport(name="a", entries=[], counts={"GONE": 1}) + orphaned = ProfileReport(name="pcem", entries=[], counts={}, + orphaned=["upstream_commit 25b54e3f"]) + moved = ProfileReport(name="m", entries=[], counts={}, + moved_repos=["a/b answers as c/b"]) + for report in (orphaned, moved): + with self.subTest(name=report.name): + self.assertGreater(drift_score(report, None, 0), drift_score(gone, None, 0)) + class TestCollectCitations(unittest.TestCase): def test_files_refs_in_document_order(self): @@ -5303,6 +5328,24 @@ class TestEmulatorsDirStatus(unittest.TestCase): self.assertTrue(profile_sync.emulators_dir_is_dirty(root, ["a"])) self.assertFalse(profile_sync.emulators_dir_is_dirty(root, ["b"])) + def test_the_refusal_names_the_changed_profile(self): + """One profile changed by another session among ~375 refused the + whole run without saying which.""" + with tempfile.TemporaryDirectory() as root: + self._git("init", "-q", cwd=root) + for name in "abc": + Path(root, f"{name}.yml").write_text(f"emulator: {name}\n") + self._git("add", "a.yml", "b.yml", "c.yml", cwd=root) + self._git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "p", cwd=root) + Path(root, "b.yml").write_text("emulator: b2\n") + self.assertEqual(profile_sync.emulators_dir_is_dirty(root, ["a", "b", "c"]), + ["b.yml"]) + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit): + profile_sync._refuse_dirty_tree(root, ["a", "b", "c"]) + self.assertIn("b.yml has uncommitted changes", err.getvalue()) + self.assertNotIn("a.yml", err.getvalue()) + def test_a_tree_outside_any_repository_is_not_clean(self): with tempfile.TemporaryDirectory() as root, mock.patch.dict( os.environ, {"GIT_CEILING_DIRECTORIES": str(Path(root).parent)} @@ -5435,7 +5478,7 @@ class UpstreamForkedByThePort(unittest.TestCase): upstream with the fork. Compared to upstream's own HEAD, a bump moved it past anything the port carries.""" - def _views(self, base): + def _views(self, base, ancestry=None): from unittest import mock # noqa: PLC0415 fork = profile_sync.upstream.make_repo("github.com", "f", "port") @@ -5447,7 +5490,9 @@ class UpstreamForkedByThePort(unittest.TestCase): ] with mock.patch.object( profile_sync.upstream, "fork_base", return_value=base - ) as asked: + ) as asked, mock.patch.object( + profile_sync.upstream, "ancestry", return_value=ancestry + ): followed = profile_sync._follow_fork_base(views, "cache", True) return followed, asked @@ -5462,6 +5507,17 @@ class UpstreamForkedByThePort(unittest.TestCase): views, _ = self._views(None) self.assertEqual(views[1].head, "up-tip") + def test_a_pin_newer_than_the_base_keeps_the_tip(self): + """melonds documents the Qt standalone from melonDS-emu/melonDS: its + upstream_commit is 825 commits past the merge base with the libretro + fork, and reading its refs at the 2021 base reported them CHANGED.""" + views, _ = self._views("old-merge", ancestry=(825, 0)) + self.assertEqual(views[1].head, "up-tip") + + def test_a_pin_behind_the_base_follows_it(self): + views, _ = self._views("new-base", ancestry=(0, 12)) + self.assertEqual(views[1].head, "new-base") + class ACachedTipOlderThanThePin(TestBuildReport): """scummvm-standalone was pinned on 6f7b060d while the cached tip was diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 9f3eebea4..a1ed98574 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -623,6 +623,21 @@ class AMissingDumpNamesTheProfileItFills(unittest.TestCase): needs = provenance_report.profile_needs(self.PROFILES, db) self.assertNotIn("dicrom.rom", needs["by_name"]) + def test_a_file_installed_with_the_emulator_is_not_a_need(self): + """43 of 50 needs were bundled entries (samcoupe.rom, zesarux's + tk90x.rom), burying the six xm7 ROMs a maintainer has to find.""" + from scripts import provenance_report # noqa: PLC0415 + + profiles = { + "samcoupe": {"files": [{"name": "samcoupe.rom", "required": True, + "bundled": True, "sha1": "f" * 40}]}, + "hcl": {"files": [{"name": "kanji.rom", "required": True, "bundled": True, + "not_bundled_on": ["batocera"], "sha1": "9" * 40}]}, + } + needs = provenance_report.profile_needs(profiles, {"files": {}}) + self.assertNotIn(("sha1", "f" * 40), needs["by_hash"]) + self.assertIn(("sha1", "9" * 40), needs["by_hash"]) + class TheWantedPageAsksOnlyForWhatCannotBeMade(unittest.TestCase): """The site's Wanted page asked the public for 79 dumps the collection diff --git a/tests/test_split_pack.py b/tests/test_split_pack.py index 497380d2b..2ae04dcde 100644 --- a/tests/test_split_pack.py +++ b/tests/test_split_pack.py @@ -14,6 +14,8 @@ parts extracted into one folder are the pack. from __future__ import annotations +import contextlib +import io import os import stat import subprocess @@ -230,6 +232,26 @@ class PartsAreNotTakenForPacks(SplitFixture): [p.name for p in parts], ) + def test_launcher_archives_are_not_packs(self): + """build_port.py writes RetroBIOS.zip, RetroBIOS-tools.zip and + RetroBIOS-batocera.zip to dist/: verify_and_finalize_packs judged + them as packs, failed them and injected a manifest into each.""" + for name in ("RetroBIOS.zip", "RetroBIOS-tools.zip", "RetroBIOS-batocera.zip"): + self._build(self.dist / name, {"RetroBIOS.sh": b"#!/bin/sh\n"}) + before = {p.name: p.read_bytes() for p in self.dist.glob("RetroBIOS*.zip")} + self.assertEqual(builder._pack_archives(str(self.dist)), [self.pack.name]) + # A --system platform pack carries its system after the suffix. + system = self.dist / "Demo_1.0_BIOS_Pack_Playstation.zip" + self._build(system, {"one.bin": b"1"}) + self.assertEqual(builder._pack_archives(str(self.dist)), + sorted([self.pack.name, system.name])) + system.unlink() + with contextlib.redirect_stdout(io.StringIO()): + builder.verify_and_finalize_packs(str(self.dist), {"files": {}, "indexes": {}}, + skip_conformance=True) + self.assertEqual({p.name: p.read_bytes() for p in self.dist.glob("RetroBIOS*.zip")}, + before) + def test_the_directory_form_splits_only_what_is_too_large(self): small = self.dist / "Small_BIOS_Pack.zip" self._build(small, {"one.bin": b"1"}) diff --git a/tests/test_system_selection.py b/tests/test_system_selection.py index 8623c9c08..8ab0fe7cd 100644 --- a/tests/test_system_selection.py +++ b/tests/test_system_selection.py @@ -424,6 +424,67 @@ class ASystemPackHoldsThatSystem(unittest.TestCase): self.assertEqual(report["total_files"], 2) +class ATreeHoldsAPathForAnotherProfile(unittest.TestCase): + """--system colecovision --standalone: bluemsx keeps no file of the + system but its Machines/ tree is packed, and holds msx-emu's coleco.rom + path. verify skipped the tree of a profile left without files and + credited msx-emu's copy the pack never wrote.""" + + def test_report_and_pack_agree(self): + import hashlib + import io + import shutil + from contextlib import redirect_stdout + + import yaml + from verify import verify_emulator + + common._emulator_profiles_cache.clear() + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + previous = os.getcwd() + os.chdir(root) + self.addCleanup(os.chdir, previous) + data = b"coleco" * 4 + sha1 = hashlib.sha1(data).hexdigest() + rel = "bios/Coleco/ColecoVision/coleco.rom" + Path(rel).parent.mkdir(parents=True) + Path(rel).write_bytes(data) + tree = Path("data/tree/Machines/COL") + tree.mkdir(parents=True) + (tree / "coleco.rom").write_bytes(data) + db = {"files": {sha1: {"path": rel, "name": "coleco.rom", "size": len(data), + "md5": hashlib.md5(data).hexdigest(), "crc32": ""}}, + "indexes": {"by_md5": {}, "by_name": {"coleco.rom": [sha1]}, + "by_crc32": {}, "by_path_suffix": {}}} + Path("platforms").mkdir() + Path("platforms/_data_dirs.yml").write_text(yaml.safe_dump( + {"data_directories": {"tree": {"local_cache": "data/tree"}}})) + Path("emulators").mkdir() + Path("emulators/atree.yml").write_text(yaml.safe_dump({ + "emulator": "ATree", "type": "libretro", "cores": ["atree"], + "systems": ["colecovision", "msx"], + "data_directories": [{"ref": "tree"}], + "files": [{"name": "msx.rom", "system": "msx"}], + })) + Path("emulators/msxemu.yml").write_text(yaml.safe_dump({ + "emulator": "MsxEmu", "type": "libretro", "cores": ["msxemu"], + "systems": ["colecovision"], + "files": [{"name": "coleco.rom", "path": "Machines/COL/coleco.rom"}], + })) + names = ["atree", "msxemu"] + report = verify_emulator(names, "emulators", db, platforms_dir="platforms", + system_ids=["colecovision"]) + record = builder.PackRecord() + with redirect_stdout(io.StringIO()): + builder.generate_emulator_pack( + names, "emulators", db, "bios", "", offline=True, + platforms_dir="platforms", record=record, system_ids=["colecovision"], + ) + self.assertEqual(sorted(report["pack_destinations"]), sorted(record.files)) + + class ListingsKeepTheirColumns(unittest.TestCase): """--list-systems and --list-emulators started their last column at a position that moved with the file count and the profile type.""" diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index 872d1302d..7f043a22b 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -5,6 +5,7 @@ from __future__ import annotations import contextlib import io import os +import re import subprocess import sys import tempfile @@ -84,6 +85,22 @@ class AMistypedTargetGetsTheClosestName(unittest.TestCase): +class AMistypedTargetAcrossAllPlatforms(unittest.TestCase): + """`verify --all --target swich` ended on "no platform has it" after 17 + INFO lines, two of which held the suggestion.""" + + def test_the_refusal_suggests_and_names_the_platforms(self): + platforms = common.list_registered_platforms(str(REPO_ROOT / "platforms")) + out = io.StringIO() + with contextlib.redirect_stdout(out), self.assertRaises(ValueError) as caught: + common.build_target_cores_cache( + platforms, "swich", str(REPO_ROOT / "platforms"), is_all=True + ) + self.assertIn("Did you mean: switch (", str(caught.exception)) + self.assertIn("retroarch", str(caught.exception)) + self.assertEqual(out.getvalue(), "") + + class AMistypedDataKeyGetsTheClosestName(unittest.TestCase): """`--key dolphin-sis` printed all 27 keys on one line.""" @@ -635,5 +652,45 @@ class TheGitHubTokenComesFromGh(unittest.TestCase): run.assert_not_called() +class CitedCommandsRunWhenPasted(unittest.TestCase): + """check_freshness wrote "run python scripts/generate_truth.py" on one + line and "export_native.py --platform bizhawk writes it again" two lines + below: pasted from the repository root, the second is command not found. + A message that hands the reader a command spells the whole command.""" + + CITED = re.compile(r"(?:^|run |with |then |, )([a-z_]+\.py) --[a-z]") + + def test_no_message_cites_a_bare_script(self): + import ast # noqa: PLC0415 + + offenders = [] + for path in sorted((REPO_ROOT / "scripts").rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + docs = { + id(node.body[0].value) + for node in ast.walk(tree) + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, + ast.AsyncFunctionDef)) + and node.body and isinstance(node.body[0], ast.Expr) + } + offenders += [ + f"{path.relative_to(REPO_ROOT)}:{node.lineno} {node.value.strip()[:60]}" + for node in ast.walk(tree) + if isinstance(node, ast.Constant) and isinstance(node.value, str) + and id(node) not in docs and self.CITED.search(node.value) + ] + self.assertEqual(offenders, []) + + def test_the_target_hint_is_a_whole_command(self): + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + targets = Path(tmp) / "targets" + targets.mkdir() + (targets / "p.yml").write_text("targets:\n switch:\n cores: [a]\n") + with self.assertRaises(ValueError) as caught: + common.load_target_config("p", "swich", tmp) + self.assertIn("python scripts/verify.py --platform p --list-targets", + str(caught.exception)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validated_choice.py b/tests/test_validated_choice.py index ca7198b3c..f9aba6707 100644 --- a/tests/test_validated_choice.py +++ b/tests/test_validated_choice.py @@ -325,5 +325,55 @@ class AlternativesAreLookedUpBySlot(unittest.TestCase): self.assertEqual(built.call_count, 2) +class AVersionTakenWithoutACheck(unittest.TestCase): + """The Batocera pack printed two DISCREPANCY lines for BLOOD.RFF that + verify never showed: Raze lists three sized versions and a fourth, + "Unknown Version", it takes without a check (the shareware RFF), and + the message named one accepted size of the three.""" + + RAZE = {"files": [ + {"name": "BLOOD.RFF", "path": "BLOOD.RFF", "size": 8200353, + "crc32": "b291418f", "validation": ["size", "crc32"]}, + {"name": "BLOOD.RFF", "path": "BLOOD.RFF", "size": 9570681, + "crc32": "a8fdda84", "validation": ["size", "crc32"]}, + {"name": "BLOOD.RFF", "path": "BLOOD.RFF"}, + ]} + + def _file(self, tmp: str) -> str: + path = Path(tmp) / "BLOOD.RFF" + path.write_bytes(b"shareware") + return str(path) + + def test_the_unchecked_version_accepts_the_copy(self): + from validation import check_file_validation # noqa: PLC0415 + + index = _build_validation_index({"raze": self.RAZE}) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + self.assertIsNone(check_file_validation( + self._file(tmp), "BLOOD.RFF", index, emulators={"raze"} + )) + + def test_the_pack_reads_it_as_an_alternative(self): + import generate_pack # noqa: PLC0415 + + fe = {"name": "BLOOD.RFF", "source_profile": "raze", "source_path": "BLOOD.RFF"} + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + self.assertTrue(generate_pack._matches_an_alternative( + fe, self._file(tmp), {"raze": self.RAZE} + )) + + def test_a_refusal_names_every_accepted_size(self): + from validation import check_file_validation # noqa: PLC0415 + + checked = {"files": self.RAZE["files"][:2]} + index = _build_validation_index({"raze": checked}) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + reason, emulators = check_file_validation( + self._file(tmp), "BLOOD.RFF", index, emulators={"raze"} + ) + self.assertEqual(emulators, ["raze"]) + self.assertEqual(reason, "size mismatch: got 9, accepted [8200353,9570681]") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index 82b25229b..c825b52a5 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -200,5 +200,35 @@ class EveryGapCountedIsNamed(unittest.TestCase): self.assertIn("MISSING (game data): engine needs level.dat", verbose.getvalue()) +class CoreCopiesWithoutAGapEntryAreCounted(unittest.TestCase): + """verify --platform recalbox --system nec-pc-engine printed no Core + files line while the pack held six geargrafx syscard copies: the gap + report was empty, and the section returned before reading the pack's + count.""" + + def test_the_packed_count_is_printed(self): + import contextlib # noqa: PLC0415 + import io # noqa: PLC0415 + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import verify # noqa: PLC0415 + + for packed, expected in (({"bios": 6, "game_data": 0}, "Core files: 6 in pack"), + ({"bios": 0, "game_data": 3}, "Game data: 3 in pack")): + out = io.StringIO() + with contextlib.redirect_stdout(out): + verify._print_undeclared_section( + {"undeclared_files": [], "packed_extras": packed}, verbose=False + ) + with self.subTest(packed=packed): + self.assertIn(expected, out.getvalue()) + out = io.StringIO() + with contextlib.redirect_stdout(out): + verify._print_undeclared_section( + {"undeclared_files": [], "packed_extras": {"bios": 0, "game_data": 0}}, False + ) + self.assertEqual(out.getvalue(), "") + + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index 7a8ccc108..bd63cb157 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -522,7 +522,8 @@ reports the profile as uncheckable, and the CI job that runs | `version_basis` | every profile (`tests/test_version_basis.py`) | where `core_version` is read, by `kind`: `file` (`path`, `pattern`, optional `format`, and `url` with its `commit` for a file outside the profile's repositories, such as the package file of a build a platform ships), `tag` (the nearest `tag` in the pin's history and its `distance`, `core_version` in git describe's form, `match` the glob a tag must fit), `alias` (`of`), `package` (`artifact`, the APK or binary read, `url` where it was obtained), `none` (the code states no version: `core_version` null) or `unverified` (no reading could be made: `core_version` stays as written and is reported unconfirmed, `why` says why: `no_pin`, `host_unsupported`, `host_gone`, `host_unreachable`, `pin_off_branch`, `no_artifact`, `no_source`). `repo` names `upstream` or `source_mirror` when the version is that tree's. `profile_sync --verify-version` reads it again, and a pin advance rewrites `core_version` from it | | `source_commit` | see above | commit SHA of `source` the code was read at. Anchors every `source_ref` line number | | `upstream_commit` | no | same, for `upstream` | -| `pin_frozen` | no | why `source_commit` is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag | +| `pin_frozen` | with `pin_build` | why `source_commit` is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag | +| `pin_build` | no | the build file each platform shipping a `pin_frozen` build reads its revision from: a list of `platform` (registry slug), `repo` (the platform's own repository), `revision` (the tag or commit the file was read at), `path`, `variable` (`PKG_VERSION`, `_VERSION`, `rp_module_repo`) and `value` (what it held there). `check_freshness.py` reads the file again where the platform's YAML now stands and reports the profile `STALE` when the platform builds another revision. A hold that no build file names (a release binary, an abandoned repository) carries `pin_frozen` alone | | `pin_follows` | no | why `source_commit` follows the branch tip when a single platform ships the profile: that platform builds the tip, cited by its build file. A bump refuses a profile one platform ships until it carries `pin_frozen` or `pin_follows` | | `data_root` | no | per operating system (`android`, `ios`, `windows`, `macos`, `linux`), the folder the app reads its user data from, under which `standalone_path` destinations sit: `path` as the user finds it (`Android/data//files`, `%APPDATA%/...`), with the `source_ref` that builds it. Only the systems the app runs on are keyed | | `upstream_frozen` | no | why `upstream_commit` is held: the port vendors that upstream tree, synced to that release, or merged that commit. The upstream view is read at its own pin and a bump leaves it. Without it, a port that is a GitHub fork of its upstream follows the merge base of the two | diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index d2c28dbc0..ceaa396d0 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -39,8 +39,9 @@ tests are reported as skipped, so install them before a pull request (`pip install "jsonschema[format-nongpl]" markdown`). The installer's full-screen tests drive `install.py --tui` on a pseudo-terminal and read the screen back with `pyte`, and fuzz the key decoder with `hypothesis`: both skip -without them (`pip install pyte hypothesis`). No test framework beyond the -standard library `unittest` module. +without them (`pip install pyte hypothesis`). `pip install -e ".[dev]"` from the +checkout installs all of these at once, from the `dev` extra of `pyproject.toml`. +No test framework beyond the standard library `unittest` module. ## Modules at a glance @@ -64,7 +65,7 @@ Every module in `tests/`, by the area it guards. | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` | ## Test architecture diff --git a/wiki/tools.md b/wiki/tools.md index 6cce3c211..5e4359e12 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -52,7 +52,7 @@ Pipeline steps: | 4c | Generate target manifests | `--skip-packs` | | 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - | | 4e | Games index: what the installer reads for the games on a card (`generate_games_index.py`, fails under its coverage floor) | - (reads the committed install lists) | -| 5 | Consistency check: counts, paths per archive, every emulator pack | `--skip-packs`, or no full pack built (`--source platform` or `truth`) | +| 5 | Consistency check: counts, paths per archive, every emulator and system pack | `--skip-packs`, or no full pack built (`--source platform` or `truth`) | | 6 | Pack integrity (extract + hash) | `--skip-packs`, or no full pack built | | 7 | Generate README and CONTRIBUTING, then the installer page's tables and run samples (`generate_installer_doc.py`) | `--skip-docs` | | 8 | Generate site | `--skip-docs` | @@ -91,8 +91,9 @@ alone, run `python scripts/check_buildbot_system.py`. - **verify.py and generate_pack.py read the same functions.** A count or a path that differs between the report and the pack is a defect, which is what step 5 checks: the counts, the paths each archive holds against the - ones verify credits (`--destinations`), and every emulator pack, each - profile in each build, built in memory against `verify --emulator`. + ones verify credits (`--destinations`), every emulator pack, each + profile in each build, built in memory against `verify --emulator`, and + every system pack that gathers several profiles against `verify --system`. Equal counts once hid four RetroArch paths credited and never written. - **Generators write through `write_if_changed()`.** A new timestamp alone rewrites nothing, so a diff on a generated file always means content. @@ -694,7 +695,15 @@ NAME` reads the folders again. The `profiles` area always checks `website/src/data/source-paths.json` against the profiles, offline: a recorded tree that no profile pins any more -is `STALE` until `sitesources.py --fetch` runs again. +is `STALE` until `sitesources.py --fetch` runs again. It also reads each +`pin_build` reference of a held pin: where the platform YAML was scraped at +the reference's own revision the row is `OK` without network, otherwise the +build file is fetched at the platform's new revision (or at the branch head +of a platform this repository does not scrape) and its variable compared +with the recorded value. A different value is `STALE`: the platform now +builds another revision, and the pin, the refs and `pin_build` move with it. +Profiles with `pin_frozen` and no `pin_build` are counted on one `SKIPPED` +row, since no build file names their revision. ```bash python scripts/check_freshness.py