feat: release packs for marked platforms only

release_pack: true in the registry is the list a release builds: RetroArch,
EmuDeck, RetroBat, Batocera, RomM and MiSTer, 83% of the downloads of
v2026.10.10. generate_pack --all, --verify-packs, the pipeline consistency
step, release_record and check_release read it. Manifests are still written
for every platform.
This commit is contained in:
Abdessamad Derraz committed 2026-10-23 17:06:46 +02:00
1 parent d458cf653a
commit 32c648b67d
10 files changed
+216 -24

No files matched your search

+10
View File
@@ -177,10 +177,15 @@ _standalone_copies: &standalone_copies
android:
- $EXTERNAL_STORAGE/Android/data/com.flycast.emulator/files/data
# release_pack: true marks the platforms a release publishes a ZIP for, the
# ones most people download (download_count of v2026.10.10, part 1 of each
# pack). Every other platform is served by the installer, whose manifest is
# written for all of them. wiki/release-process.md says how to revisit it.
platforms:
retroarch:
config: retroarch.yml
status: active
release_pack: true
device_group: computer
extract_to: system/
logo: https://raw.githubusercontent.com/libretro/RetroArch/master/media/retroarch-vector_invader-only.svg
@@ -486,6 +491,7 @@ platforms:
batocera:
config: batocera.yml
status: active
release_pack: true
device_group: gaming-os
extract_to: /userdata/bios/
logo: https://raw.githubusercontent.com/batocera-linux/batocera-emulationstation/master/resources/splash_batocera.svg
@@ -561,6 +567,7 @@ platforms:
retrobat:
config: retrobat.yml
status: active
release_pack: true
device_group: computer
extract_to: bios/
logo: https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png
@@ -608,6 +615,7 @@ platforms:
emudeck:
config: emudeck.yml
status: active
release_pack: true
device_group: computer
extract_to: ~/Emulation/bios/
logo: https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/icons/EmuDeck.png
@@ -749,6 +757,7 @@ platforms:
romm:
config: romm.yml
status: active
release_pack: true
device_group: server
extract_to: library/bios/
logo: https://avatars.githubusercontent.com/u/164058018
@@ -1024,6 +1033,7 @@ platforms:
misterfpga:
config: misterfpga.yml
status: active
release_pack: true
device_group: fpga
extract_to: /media/fat/games/
logo: https://avatars.githubusercontent.com/u/29345272
+32
View File
@@ -0,0 +1,32 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/registry.schema.json",
"title": "RetroBIOS platform registry",
"description": "platforms/_registry.yml: each registered platform, its status, and whether a release publishes a ZIP pack for it. Install detection and the other per-platform fields are read by their own tools.",
"type": "object",
"required": ["platforms"],
"properties": {
"platforms": {
"type": "object",
"minProperties": 1,
"propertyNames": { "pattern": "^[a-z0-9_]+$" },
"additionalProperties": {
"type": "object",
"required": ["config", "status"],
"properties": {
"config": { "type": "string", "pattern": "\\.yml$" },
"status": {
"description": "active: scraped and released. archived: the configuration is kept, no scheduled scrape.",
"enum": ["active", "archived"]
},
"release_pack": {
"description": "true when a release publishes the platform's full ZIP pack. Absent or false: the installer serves the platform, its manifest is written all the same.",
"type": "boolean"
},
"device_group": { "type": "string" },
"extract_to": { "type": ["string", "null"] }
}
}
}
}
}
+26 -2
View File
@@ -5,7 +5,9 @@ Run on the draft before publishing it, and on the published release at any
time. Every pack asset release.json names must be on the release at the sha256
SHA256SUMS.txt lists for it, as must the handheld port (RetroBIOS.zip) when
the list names it, the parts of a pack must add up to the download
size the README prints, and nothing else may be attached. An upload that
size the README prints, and nothing else may be attached. The record holds
one pack for each platform the registry marks release_pack, and no other
pack. An upload that
failed in the loop, or a part re-uploaded from an older build, otherwise
publishes a release that `sha256sum --check` rejects for every user.
@@ -29,6 +31,7 @@ from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
from build_port import PORT_ZIP
from check_release_assets import digests_of
from common import list_release_pack_platforms
from largefiles import LARGE_FILES_REPO
from release_record import RECORD, load_record
@@ -69,15 +72,31 @@ def compare_release(
assets: dict[str, Asset],
tag: str,
local_files: dict[str, str] | None = None,
released: set[str] | None = None,
) -> list[Finding]:
"""Every way the release can differ from what was built.
*local_files* maps a file kept beside the packs (the checksum list, its
signature) to its local sha256: the copy on the release must be that one.
*released* names the platforms the registry marks release_pack: each one
has a recorded pack, and no recorded pack serves only other platforms.
"""
findings: list[Finding] = []
if record.get("tag") != tag:
findings.append(Finding("tag", tag, f"release.json records {record.get('tag')}"))
if released is not None:
packs = record.get("packs", {})
served = {p for pack in packs.values() for p in pack.get("platforms", ())}
for platform in sorted(released - served):
findings.append(Finding(
"no-pack", platform, "release_pack: true, no pack in release.json"
))
for pack_name, pack in sorted(packs.items()):
if not set(pack.get("platforms", ())) & released:
findings.append(Finding(
"not-released", pack_name,
"serves no platform marked release_pack in platforms/_registry.yml",
))
recorded = {
name: pack_name
@@ -167,6 +186,10 @@ def main() -> int:
help=f"local {SUMS} the packs were listed in. Its copy and signature on "
f"the release must be these files (default: the list attached to the release)",
)
parser.add_argument(
"--platforms-dir", default="platforms",
help="registry whose release_pack platforms each need a pack (default: %(default)s)",
)
parser.add_argument("--json", action="store_true", help="JSON output")
args = parser.parse_args()
@@ -199,7 +222,8 @@ def main() -> int:
print(f"ERROR: {SUMS}: {exc}", file=sys.stderr)
return 2
findings = compare_release(record, sums, assets, tag, local_files)
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
findings = compare_release(record, sums, assets, tag, local_files, released)
if args.json:
print(json.dumps(
{"tag": tag, "draft": draft, "assets": len(assets),
+19
View File
@@ -437,6 +437,25 @@ def list_registered_platforms(
return platforms
def list_release_pack_platforms(
platforms_dir: str = "platforms",
include_archived: bool = False,
) -> list[str]:
"""Registered platforms a release publishes a ZIP pack for.
`release_pack: true` in _registry.yml is the whole list: the build of
`--all`, the pack checks, the release record and every download table
read it. The other platforms keep their install manifest and are served
by the installer.
"""
registry = load_platform_registry(platforms_dir)
return [
name
for name in list_registered_platforms(platforms_dir, include_archived)
if registry.get(name, {}).get("release_pack") is True
]
# A list this short is printed whole rather than pointed at.
SHORT_LIST = 12
+29 -14
View File
@@ -57,6 +57,7 @@ from common import (
canonical_target_name,
platform_files_target,
list_registered_platforms,
list_release_pack_platforms,
list_system_ids,
load_data_dir_registry,
load_database,
@@ -2623,7 +2624,9 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
if args.platform:
platforms = [args.platform]
elif args.all:
platforms = list_registered_platforms(
# The packs --all builds: a platform without release_pack has no pack
# to check, and its absence from the directory is no failure.
platforms = list_release_pack_platforms(
args.platforms_dir, include_archived=args.include_archived
)
else:
@@ -3112,7 +3115,9 @@ def _main(held: contextlib.ExitStack) -> None:
help="Platform name (e.g., retroarch)",
)
parser.add_argument(
"--all", action="store_true", help="Generate packs for all active platforms"
"--all", action="store_true",
help="Generate the release packs (release_pack in the registry), or with"
" --manifest the install manifests of every platform",
)
parser.add_argument(
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
@@ -3264,10 +3269,14 @@ def _main(held: contextlib.ExitStack) -> None:
)
return
if args.list:
# Archived platforms build like the others but have no release pack.
# Every platform builds with --platform. --all builds the release
# packs only, the others are served by the installer.
active = set(list_registered_platforms(args.platforms_dir))
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
for p in list_platforms(args.platforms_dir):
print(p if p in active else f"{p} (archived)")
marks = [m for m, on in (("archived", p not in active),
("installer only", p not in released)) if on]
print(f"{p} ({', '.join(marks)})" if marks else p)
return
if args.list_emulators:
list_emulator_profiles(args.emulators_dir)
@@ -3393,15 +3402,20 @@ def _main(held: contextlib.ExitStack) -> None:
system_filter = getattr(args, "system_ids", None)
# Platform mode
if args.all:
# Install manifests cover every registered platform. "Archived" means
if args.all and args.manifest:
# Install manifests cover every registered platform: the installer
# serves each one, with or without a release pack. "Archived" means
# upstream is no longer scraped on a schedule, not that the platform
# lost its users: its packs still ship, so the installer still needs
# its file list.
platforms = list_registered_platforms(
args.platforms_dir,
include_archived=args.include_archived or args.manifest,
# lost its users.
platforms = list_registered_platforms(args.platforms_dir, include_archived=True)
elif args.all:
# A release publishes the packs the registry marks release_pack and
# nothing else: a pack built for another platform is one nobody uploads.
platforms = list_release_pack_platforms(
args.platforms_dir, include_archived=args.include_archived
)
if not platforms:
parser.error("no registered platform has release_pack: true")
elif args.platform:
platforms = [args.platform]
else:
@@ -4789,10 +4803,11 @@ def _expected_pack_names(
}
for p in platforms
}
# Groups depend on which platforms a run built: --all alone or with
# --include-archived. Both compositions are names a pack can carry.
# Groups depend on which platforms a run built: the release packs of
# --all, alone or with --include-archived. Both compositions are names a
# pack can carry.
for include_archived in (False, True):
pool = list_registered_platforms(
pool = list_release_pack_platforms(
platforms_dir, include_archived=include_archived
)
cache = None
+27 -3
View File
@@ -317,8 +317,27 @@ def _match_key(label: str) -> set[str]:
} - {""}
def check_consistency(verify_output: str, pack_output: str) -> bool:
"""Verify that check counts match between verify and pack for each platform."""
def release_pack_keys(platforms_dir: str = "platforms", include_archived: bool = False) -> set[str]:
"""Comparable identities of the platforms --all builds a pack for: their
registry ids and display names, normalized as _match_key does."""
from common import list_release_pack_platforms, load_platform_config # noqa: PLC0415
keys: set[str] = set()
for name in list_release_pack_platforms(platforms_dir, include_archived):
keys |= _match_key(name)
keys |= _match_key(str(load_platform_config(name, platforms_dir).get("platform", name)))
return keys
def check_consistency(
verify_output: str, pack_output: str, released: set[str] | None = None,
) -> bool:
"""Verify that check counts match between verify and pack for each platform.
*released* holds the identities of the platforms with a release pack
(release_pack_keys). verify judges every platform, the build only those:
a platform outside it has no pack to compare, which is no failure.
"""
v = parse_verify_counts(verify_output)
p = parse_pack_counts(pack_output)
excluded = parse_pack_exclusions(pack_output)
@@ -377,6 +396,8 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
f"({p_excluded} unsafe excluded) != verify {v_ok}/{v_total}"
)
all_ok = False
elif released is not None and not (v_names & released):
print(f" {v_label}: {v_ok}/{v_total} (no release pack, served by the installer)")
else:
print(f" {v_label}: {v_ok}/{v_total} (no separate pack)")
all_ok = False
@@ -1023,7 +1044,10 @@ def main() -> None:
skip_notice("consistency", "no full pack built")
results["consistency"] = SKIPPED
elif pack_output and verify_output:
ok = check_consistency(verify_output, pack_output)
ok = check_consistency(
verify_output, pack_output,
release_pack_keys(include_archived=args.include_archived),
)
# Counts can agree while the paths differ, both ways at once.
ok = check_pack_destinations(destinations_file, pack_output) and ok
ok = check_emulator_packs() and ok
+37 -3
View File
@@ -26,11 +26,15 @@ import split_pack
from common import (
ArtifactLockBusy,
artifact_lock,
list_registered_platforms,
list_release_pack_platforms,
write_text_atomic,
)
RECORD = "release.json"
# Each registered platform has a page at get/<platform>/ on the site: the
# current pack's files, or the command when no release publishes one. The
# address outlives every release, so a link written elsewhere keeps working.
GET_DIR = "get"
class UnfinishedSplitError(Exception):
@@ -46,11 +50,12 @@ def release_owners(platforms_dir: str = "platforms") -> dict[str, list[str]]:
Rebuilt from the name builder, never matched by substring: matched by
substring, RetroArch_v1.22.2_AndroidArm64V8A_BIOS_Pack.zip sorts before
the full pack and was taken for the RetroArch pack.
the full pack and was taken for the RetroArch pack. Only the platforms
marked release_pack own a release pack.
"""
import generate_pack
platforms = list_registered_platforms(platforms_dir, include_archived=True)
platforms = list_release_pack_platforms(platforms_dir, include_archived=True)
owners: dict[str, list[str]] = {}
for platform, names in generate_pack._expected_pack_names(
platforms, platforms_dir, None, None
@@ -127,6 +132,35 @@ def load_record(path: str | Path = RECORD) -> dict:
return {}
def get_page(platform: str, site: str = "") -> str:
"""The permanent download address of a platform, under *site*."""
return f"{site}{GET_DIR}/{platform}/"
def advertised(record: dict, platforms_dir: str = "platforms") -> dict:
"""The record as the download tables show it: release_pack platforms only.
A release cut before a platform lost its release_pack still holds that
platform's pack. The tables follow the registry, so a platform left out
is offered the command, and a pack two platforms shared names the one
still released.
"""
if not record:
return {}
released = set(list_release_pack_platforms(platforms_dir, include_archived=True))
packs = {}
for name, pack in record.get("packs", {}).items():
serves = [p for p in pack.get("platforms", ()) if p in released]
if serves:
packs[name] = {**pack, "platforms": serves}
return {**record, "packs": packs}
def load_advertised(path: str | Path = RECORD, platforms_dir: str = "platforms") -> dict:
"""The committed record, reduced to the platforms a release publishes."""
return advertised(load_record(path), platforms_dir)
def pack_name_for(platform: str, record: dict) -> str | None:
"""Name of the release pack that serves a platform, if it has one."""
for name, pack in sorted(record.get("packs", {}).items()):
+26
View File
@@ -391,6 +391,31 @@ def _save_folder_once_paths(path: Path, document: dict) -> list[str]:
]
def _registry_contract() -> list[str]:
"""The registry against its schema, and a release that still ships a pack.
`release_pack` is the only list of the ZIPs a release publishes: a
registry where no active platform carries it would cut a release with
no pack at all, and every download link would lead to the command.
"""
registry_path = ROOT / "platforms" / "_registry.yml"
if not registry_path.is_file():
return []
with registry_path.open(encoding="utf-8") as handle:
registry = yaml_load(handle) or {}
label = str(registry_path.relative_to(ROOT))
out = _errors(_validator("registry.schema.json"), registry, label)
platforms = registry.get("platforms") or {}
if isinstance(platforms, dict) and not any(
isinstance(entry, dict)
and entry.get("release_pack") is True
and entry.get("status", "active") == "active"
for entry in platforms.values()
):
out.append(f"{label}: no active platform has release_pack: true")
return out
def _save_directory_blocks() -> list[str]:
"""Each registry save_directory is null or a folder and a by_core flag.
@@ -572,6 +597,7 @@ def main() -> int:
errors.extend(_unknown_data_dir_refs())
errors.extend(_repository_layout_in_profiles())
errors.extend(_save_directory_blocks())
errors.extend(_registry_contract())
if not args.source_only:
database_path = ROOT / "database.json"
+2 -1
View File
@@ -346,7 +346,8 @@ class ListPlatformsTakesTheSharedFlag(unittest.TestCase):
)
self.assertEqual(proc.returncode, 0, proc.stderr)
lines = proc.stdout.splitlines()
self.assertIn("retropie (archived)", lines)
self.assertIn("retropie (archived, installer only)", lines)
self.assertIn("knulli (installer only)", lines)
self.assertIn("retroarch", lines)
+8 -1
View File
@@ -140,12 +140,19 @@ class VerifyPackChoice(unittest.TestCase):
self._run(self._args())
self.assertEqual(out.getvalue().strip().splitlines()[-1], "Verified 1 pack: all OK")
def test_include_archived_reaches_the_archived_pack(self):
def test_the_sweep_checks_release_packs_and_a_named_platform_its_own(self):
"""--all checks the packs a release publishes (release_pack): the
RetroPie pack, archived and served by the installer, is left to a
check that names it."""
stem = self.gp._platform_pack_stem(["retropie"], "retropie", "platforms")
self._touch(f"{stem}_BIOS_Pack.zip")
self._touch(f"{self.stem}_BIOS_Pack.zip")
checked = self._run(
self._args(platform=None, all=True, include_archived=True)
)
self.assertIn(("retroarch", f"{self.stem}_BIOS_Pack.zip"), checked)
self.assertNotIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked)
checked = self._run(self._args(platform="retropie"))
self.assertIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked)