diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 90d660fd..cb9d2d9a 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -177,10 +177,15 @@ _standalone_copies: &standalone_copies android: - $EXTERNAL_STORAGE/Android/data/com.flycast.emulator/files/data +# release_pack: true marks the platforms a release publishes a ZIP for, the +# ones most people download (download_count of v2026.10.10, part 1 of each +# pack). Every other platform is served by the installer, whose manifest is +# written for all of them. wiki/release-process.md says how to revisit it. platforms: retroarch: config: retroarch.yml status: active + release_pack: true device_group: computer extract_to: system/ logo: https://raw.githubusercontent.com/libretro/RetroArch/master/media/retroarch-vector_invader-only.svg @@ -486,6 +491,7 @@ platforms: batocera: config: batocera.yml status: active + release_pack: true device_group: gaming-os extract_to: /userdata/bios/ logo: https://raw.githubusercontent.com/batocera-linux/batocera-emulationstation/master/resources/splash_batocera.svg @@ -561,6 +567,7 @@ platforms: retrobat: config: retrobat.yml status: active + release_pack: true device_group: computer extract_to: bios/ logo: https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png @@ -608,6 +615,7 @@ platforms: emudeck: config: emudeck.yml status: active + release_pack: true device_group: computer extract_to: ~/Emulation/bios/ logo: https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/icons/EmuDeck.png @@ -749,6 +757,7 @@ platforms: romm: config: romm.yml status: active + release_pack: true device_group: server extract_to: library/bios/ logo: https://avatars.githubusercontent.com/u/164058018 @@ -1024,6 +1033,7 @@ platforms: misterfpga: config: misterfpga.yml status: active + release_pack: true device_group: fpga extract_to: /media/fat/games/ logo: https://avatars.githubusercontent.com/u/29345272 diff --git a/schemas/registry.schema.json b/schemas/registry.schema.json new file mode 100644 index 00000000..111facc6 --- /dev/null +++ b/schemas/registry.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/registry.schema.json", + "title": "RetroBIOS platform registry", + "description": "platforms/_registry.yml: each registered platform, its status, and whether a release publishes a ZIP pack for it. Install detection and the other per-platform fields are read by their own tools.", + "type": "object", + "required": ["platforms"], + "properties": { + "platforms": { + "type": "object", + "minProperties": 1, + "propertyNames": { "pattern": "^[a-z0-9_]+$" }, + "additionalProperties": { + "type": "object", + "required": ["config", "status"], + "properties": { + "config": { "type": "string", "pattern": "\\.yml$" }, + "status": { + "description": "active: scraped and released. archived: the configuration is kept, no scheduled scrape.", + "enum": ["active", "archived"] + }, + "release_pack": { + "description": "true when a release publishes the platform's full ZIP pack. Absent or false: the installer serves the platform, its manifest is written all the same.", + "type": "boolean" + }, + "device_group": { "type": "string" }, + "extract_to": { "type": ["string", "null"] } + } + } + } + } +} diff --git a/scripts/check_release.py b/scripts/check_release.py index 018dbc16..409ee555 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -5,7 +5,9 @@ Run on the draft before publishing it, and on the published release at any time. Every pack asset release.json names must be on the release at the sha256 SHA256SUMS.txt lists for it, as must the handheld port (RetroBIOS.zip) when the list names it, the parts of a pack must add up to the download -size the README prints, and nothing else may be attached. An upload that +size the README prints, and nothing else may be attached. The record holds +one pack for each platform the registry marks release_pack, and no other +pack. An upload that failed in the loop, or a part re-uploaded from an older build, otherwise publishes a release that `sha256sum --check` rejects for every user. @@ -29,6 +31,7 @@ from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from build_port import PORT_ZIP from check_release_assets import digests_of +from common import list_release_pack_platforms from largefiles import LARGE_FILES_REPO from release_record import RECORD, load_record @@ -69,15 +72,31 @@ def compare_release( assets: dict[str, Asset], tag: str, local_files: dict[str, str] | None = None, + released: set[str] | None = None, ) -> list[Finding]: """Every way the release can differ from what was built. *local_files* maps a file kept beside the packs (the checksum list, its signature) to its local sha256: the copy on the release must be that one. + *released* names the platforms the registry marks release_pack: each one + has a recorded pack, and no recorded pack serves only other platforms. """ findings: list[Finding] = [] if record.get("tag") != tag: findings.append(Finding("tag", tag, f"release.json records {record.get('tag')}")) + if released is not None: + packs = record.get("packs", {}) + served = {p for pack in packs.values() for p in pack.get("platforms", ())} + for platform in sorted(released - served): + findings.append(Finding( + "no-pack", platform, "release_pack: true, no pack in release.json" + )) + for pack_name, pack in sorted(packs.items()): + if not set(pack.get("platforms", ())) & released: + findings.append(Finding( + "not-released", pack_name, + "serves no platform marked release_pack in platforms/_registry.yml", + )) recorded = { name: pack_name @@ -167,6 +186,10 @@ def main() -> int: help=f"local {SUMS} the packs were listed in. Its copy and signature on " f"the release must be these files (default: the list attached to the release)", ) + parser.add_argument( + "--platforms-dir", default="platforms", + help="registry whose release_pack platforms each need a pack (default: %(default)s)", + ) parser.add_argument("--json", action="store_true", help="JSON output") args = parser.parse_args() @@ -199,7 +222,8 @@ def main() -> int: print(f"ERROR: {SUMS}: {exc}", file=sys.stderr) return 2 - findings = compare_release(record, sums, assets, tag, local_files) + released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True)) + findings = compare_release(record, sums, assets, tag, local_files, released) if args.json: print(json.dumps( {"tag": tag, "draft": draft, "assets": len(assets), diff --git a/scripts/common.py b/scripts/common.py index 4b603ac4..eed59b06 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -437,6 +437,25 @@ def list_registered_platforms( return platforms +def list_release_pack_platforms( + platforms_dir: str = "platforms", + include_archived: bool = False, +) -> list[str]: + """Registered platforms a release publishes a ZIP pack for. + + `release_pack: true` in _registry.yml is the whole list: the build of + `--all`, the pack checks, the release record and every download table + read it. The other platforms keep their install manifest and are served + by the installer. + """ + registry = load_platform_registry(platforms_dir) + return [ + name + for name in list_registered_platforms(platforms_dir, include_archived) + if registry.get(name, {}).get("release_pack") is True + ] + + # A list this short is printed whole rather than pointed at. SHORT_LIST = 12 diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 319cb8f8..5f22e6bd 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -57,6 +57,7 @@ from common import ( canonical_target_name, platform_files_target, list_registered_platforms, + list_release_pack_platforms, list_system_ids, load_data_dir_registry, load_database, @@ -2623,7 +2624,9 @@ def _run_verify_packs(args: argparse.Namespace) -> None: if args.platform: platforms = [args.platform] elif args.all: - platforms = list_registered_platforms( + # The packs --all builds: a platform without release_pack has no pack + # to check, and its absence from the directory is no failure. + platforms = list_release_pack_platforms( args.platforms_dir, include_archived=args.include_archived ) else: @@ -3112,7 +3115,9 @@ def _main(held: contextlib.ExitStack) -> None: help="Platform name (e.g., retroarch)", ) parser.add_argument( - "--all", action="store_true", help="Generate packs for all active platforms" + "--all", action="store_true", + help="Generate the release packs (release_pack in the registry), or with" + " --manifest the install manifests of every platform", ) parser.add_argument( "--emulator", "-e", help="Emulator profile name(s), comma-separated" @@ -3264,10 +3269,14 @@ def _main(held: contextlib.ExitStack) -> None: ) return if args.list: - # Archived platforms build like the others but have no release pack. + # Every platform builds with --platform. --all builds the release + # packs only, the others are served by the installer. active = set(list_registered_platforms(args.platforms_dir)) + released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True)) for p in list_platforms(args.platforms_dir): - print(p if p in active else f"{p} (archived)") + marks = [m for m, on in (("archived", p not in active), + ("installer only", p not in released)) if on] + print(f"{p} ({', '.join(marks)})" if marks else p) return if args.list_emulators: list_emulator_profiles(args.emulators_dir) @@ -3393,15 +3402,20 @@ def _main(held: contextlib.ExitStack) -> None: system_filter = getattr(args, "system_ids", None) # Platform mode - if args.all: - # Install manifests cover every registered platform. "Archived" means + if args.all and args.manifest: + # Install manifests cover every registered platform: the installer + # serves each one, with or without a release pack. "Archived" means # upstream is no longer scraped on a schedule, not that the platform - # lost its users: its packs still ship, so the installer still needs - # its file list. - platforms = list_registered_platforms( - args.platforms_dir, - include_archived=args.include_archived or args.manifest, + # lost its users. + platforms = list_registered_platforms(args.platforms_dir, include_archived=True) + elif args.all: + # A release publishes the packs the registry marks release_pack and + # nothing else: a pack built for another platform is one nobody uploads. + platforms = list_release_pack_platforms( + args.platforms_dir, include_archived=args.include_archived ) + if not platforms: + parser.error("no registered platform has release_pack: true") elif args.platform: platforms = [args.platform] else: @@ -4789,10 +4803,11 @@ def _expected_pack_names( } for p in platforms } - # Groups depend on which platforms a run built: --all alone or with - # --include-archived. Both compositions are names a pack can carry. + # Groups depend on which platforms a run built: the release packs of + # --all, alone or with --include-archived. Both compositions are names a + # pack can carry. for include_archived in (False, True): - pool = list_registered_platforms( + pool = list_release_pack_platforms( platforms_dir, include_archived=include_archived ) cache = None diff --git a/scripts/pipeline.py b/scripts/pipeline.py index f1daf953..738a9a03 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -317,8 +317,27 @@ def _match_key(label: str) -> set[str]: } - {""} -def check_consistency(verify_output: str, pack_output: str) -> bool: - """Verify that check counts match between verify and pack for each platform.""" +def release_pack_keys(platforms_dir: str = "platforms", include_archived: bool = False) -> set[str]: + """Comparable identities of the platforms --all builds a pack for: their + registry ids and display names, normalized as _match_key does.""" + from common import list_release_pack_platforms, load_platform_config # noqa: PLC0415 + + keys: set[str] = set() + for name in list_release_pack_platforms(platforms_dir, include_archived): + keys |= _match_key(name) + keys |= _match_key(str(load_platform_config(name, platforms_dir).get("platform", name))) + return keys + + +def check_consistency( + verify_output: str, pack_output: str, released: set[str] | None = None, +) -> bool: + """Verify that check counts match between verify and pack for each platform. + + *released* holds the identities of the platforms with a release pack + (release_pack_keys). verify judges every platform, the build only those: + a platform outside it has no pack to compare, which is no failure. + """ v = parse_verify_counts(verify_output) p = parse_pack_counts(pack_output) excluded = parse_pack_exclusions(pack_output) @@ -377,6 +396,8 @@ def check_consistency(verify_output: str, pack_output: str) -> bool: f"({p_excluded} unsafe excluded) != verify {v_ok}/{v_total}" ) all_ok = False + elif released is not None and not (v_names & released): + print(f" {v_label}: {v_ok}/{v_total} (no release pack, served by the installer)") else: print(f" {v_label}: {v_ok}/{v_total} (no separate pack)") all_ok = False @@ -1023,7 +1044,10 @@ def main() -> None: skip_notice("consistency", "no full pack built") results["consistency"] = SKIPPED elif pack_output and verify_output: - ok = check_consistency(verify_output, pack_output) + ok = check_consistency( + verify_output, pack_output, + release_pack_keys(include_archived=args.include_archived), + ) # Counts can agree while the paths differ, both ways at once. ok = check_pack_destinations(destinations_file, pack_output) and ok ok = check_emulator_packs() and ok diff --git a/scripts/release_record.py b/scripts/release_record.py index 1926398f..6737b509 100644 --- a/scripts/release_record.py +++ b/scripts/release_record.py @@ -26,11 +26,15 @@ import split_pack from common import ( ArtifactLockBusy, artifact_lock, - list_registered_platforms, + list_release_pack_platforms, write_text_atomic, ) RECORD = "release.json" +# Each registered platform has a page at get// on the site: the +# current pack's files, or the command when no release publishes one. The +# address outlives every release, so a link written elsewhere keeps working. +GET_DIR = "get" class UnfinishedSplitError(Exception): @@ -46,11 +50,12 @@ def release_owners(platforms_dir: str = "platforms") -> dict[str, list[str]]: Rebuilt from the name builder, never matched by substring: matched by substring, RetroArch_v1.22.2_AndroidArm64V8A_BIOS_Pack.zip sorts before - the full pack and was taken for the RetroArch pack. + the full pack and was taken for the RetroArch pack. Only the platforms + marked release_pack own a release pack. """ import generate_pack - platforms = list_registered_platforms(platforms_dir, include_archived=True) + platforms = list_release_pack_platforms(platforms_dir, include_archived=True) owners: dict[str, list[str]] = {} for platform, names in generate_pack._expected_pack_names( platforms, platforms_dir, None, None @@ -127,6 +132,35 @@ def load_record(path: str | Path = RECORD) -> dict: return {} +def get_page(platform: str, site: str = "") -> str: + """The permanent download address of a platform, under *site*.""" + return f"{site}{GET_DIR}/{platform}/" + + +def advertised(record: dict, platforms_dir: str = "platforms") -> dict: + """The record as the download tables show it: release_pack platforms only. + + A release cut before a platform lost its release_pack still holds that + platform's pack. The tables follow the registry, so a platform left out + is offered the command, and a pack two platforms shared names the one + still released. + """ + if not record: + return {} + released = set(list_release_pack_platforms(platforms_dir, include_archived=True)) + packs = {} + for name, pack in record.get("packs", {}).items(): + serves = [p for p in pack.get("platforms", ()) if p in released] + if serves: + packs[name] = {**pack, "platforms": serves} + return {**record, "packs": packs} + + +def load_advertised(path: str | Path = RECORD, platforms_dir: str = "platforms") -> dict: + """The committed record, reduced to the platforms a release publishes.""" + return advertised(load_record(path), platforms_dir) + + def pack_name_for(platform: str, record: dict) -> str | None: """Name of the release pack that serves a platform, if it has one.""" for name, pack in sorted(record.get("packs", {}).items()): diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 165f71d1..5e002a78 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -391,6 +391,31 @@ def _save_folder_once_paths(path: Path, document: dict) -> list[str]: ] +def _registry_contract() -> list[str]: + """The registry against its schema, and a release that still ships a pack. + + `release_pack` is the only list of the ZIPs a release publishes: a + registry where no active platform carries it would cut a release with + no pack at all, and every download link would lead to the command. + """ + registry_path = ROOT / "platforms" / "_registry.yml" + if not registry_path.is_file(): + return [] + with registry_path.open(encoding="utf-8") as handle: + registry = yaml_load(handle) or {} + label = str(registry_path.relative_to(ROOT)) + out = _errors(_validator("registry.schema.json"), registry, label) + platforms = registry.get("platforms") or {} + if isinstance(platforms, dict) and not any( + isinstance(entry, dict) + and entry.get("release_pack") is True + and entry.get("status", "active") == "active" + for entry in platforms.values() + ): + out.append(f"{label}: no active platform has release_pack: true") + return out + + def _save_directory_blocks() -> list[str]: """Each registry save_directory is null or a folder and a by_core flag. @@ -572,6 +597,7 @@ def main() -> int: errors.extend(_unknown_data_dir_refs()) errors.extend(_repository_layout_in_profiles()) errors.extend(_save_directory_blocks()) + errors.extend(_registry_contract()) if not args.source_only: database_path = ROOT / "database.json" diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index 9a5731df..adfcdd59 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -346,7 +346,8 @@ class ListPlatformsTakesTheSharedFlag(unittest.TestCase): ) self.assertEqual(proc.returncode, 0, proc.stderr) lines = proc.stdout.splitlines() - self.assertIn("retropie (archived)", lines) + self.assertIn("retropie (archived, installer only)", lines) + self.assertIn("knulli (installer only)", lines) self.assertIn("retroarch", lines) diff --git a/tests/test_verify_pack_choice.py b/tests/test_verify_pack_choice.py index 0f243da4..4aac344d 100644 --- a/tests/test_verify_pack_choice.py +++ b/tests/test_verify_pack_choice.py @@ -140,12 +140,19 @@ class VerifyPackChoice(unittest.TestCase): self._run(self._args()) self.assertEqual(out.getvalue().strip().splitlines()[-1], "Verified 1 pack: all OK") - def test_include_archived_reaches_the_archived_pack(self): + def test_the_sweep_checks_release_packs_and_a_named_platform_its_own(self): + """--all checks the packs a release publishes (release_pack): the + RetroPie pack, archived and served by the installer, is left to a + check that names it.""" stem = self.gp._platform_pack_stem(["retropie"], "retropie", "platforms") self._touch(f"{stem}_BIOS_Pack.zip") + self._touch(f"{self.stem}_BIOS_Pack.zip") checked = self._run( self._args(platform=None, all=True, include_archived=True) ) + self.assertIn(("retroarch", f"{self.stem}_BIOS_Pack.zip"), checked) + self.assertNotIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked) + checked = self._run(self._args(platform="retropie")) self.assertIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked)