mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
feat: release packs for marked platforms only
release_pack: true in the registry is the list a release builds: RetroArch, EmuDeck, RetroBat, Batocera, RomM and MiSTer, 83% of the downloads of v2026.10.10. generate_pack --all, --verify-packs, the pipeline consistency step, release_record and check_release read it. Manifests are still written for every platform.
This commit is contained in:
1 parent
d458cf653a
commit
32c648b67d
10 files changed
+216
-24
No files matched your search
@@ -177,10 +177,15 @@ _standalone_copies: &standalone_copies
|
||||
android:
|
||||
- $EXTERNAL_STORAGE/Android/data/com.flycast.emulator/files/data
|
||||
|
||||
# release_pack: true marks the platforms a release publishes a ZIP for, the
|
||||
# ones most people download (download_count of v2026.10.10, part 1 of each
|
||||
# pack). Every other platform is served by the installer, whose manifest is
|
||||
# written for all of them. wiki/release-process.md says how to revisit it.
|
||||
platforms:
|
||||
retroarch:
|
||||
config: retroarch.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: computer
|
||||
extract_to: system/
|
||||
logo: https://raw.githubusercontent.com/libretro/RetroArch/master/media/retroarch-vector_invader-only.svg
|
||||
@@ -486,6 +491,7 @@ platforms:
|
||||
batocera:
|
||||
config: batocera.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: gaming-os
|
||||
extract_to: /userdata/bios/
|
||||
logo: https://raw.githubusercontent.com/batocera-linux/batocera-emulationstation/master/resources/splash_batocera.svg
|
||||
@@ -561,6 +567,7 @@ platforms:
|
||||
retrobat:
|
||||
config: retrobat.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: computer
|
||||
extract_to: bios/
|
||||
logo: https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png
|
||||
@@ -608,6 +615,7 @@ platforms:
|
||||
emudeck:
|
||||
config: emudeck.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: computer
|
||||
extract_to: ~/Emulation/bios/
|
||||
logo: https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/icons/EmuDeck.png
|
||||
@@ -749,6 +757,7 @@ platforms:
|
||||
romm:
|
||||
config: romm.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: server
|
||||
extract_to: library/bios/
|
||||
logo: https://avatars.githubusercontent.com/u/164058018
|
||||
@@ -1024,6 +1033,7 @@ platforms:
|
||||
misterfpga:
|
||||
config: misterfpga.yml
|
||||
status: active
|
||||
release_pack: true
|
||||
device_group: fpga
|
||||
extract_to: /media/fat/games/
|
||||
logo: https://avatars.githubusercontent.com/u/29345272
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/registry.schema.json",
|
||||
"title": "RetroBIOS platform registry",
|
||||
"description": "platforms/_registry.yml: each registered platform, its status, and whether a release publishes a ZIP pack for it. Install detection and the other per-platform fields are read by their own tools.",
|
||||
"type": "object",
|
||||
"required": ["platforms"],
|
||||
"properties": {
|
||||
"platforms": {
|
||||
"type": "object",
|
||||
"minProperties": 1,
|
||||
"propertyNames": { "pattern": "^[a-z0-9_]+$" },
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"required": ["config", "status"],
|
||||
"properties": {
|
||||
"config": { "type": "string", "pattern": "\\.yml$" },
|
||||
"status": {
|
||||
"description": "active: scraped and released. archived: the configuration is kept, no scheduled scrape.",
|
||||
"enum": ["active", "archived"]
|
||||
},
|
||||
"release_pack": {
|
||||
"description": "true when a release publishes the platform's full ZIP pack. Absent or false: the installer serves the platform, its manifest is written all the same.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"device_group": { "type": "string" },
|
||||
"extract_to": { "type": ["string", "null"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@ Run on the draft before publishing it, and on the published release at any
|
||||
time. Every pack asset release.json names must be on the release at the sha256
|
||||
SHA256SUMS.txt lists for it, as must the handheld port (RetroBIOS.zip) when
|
||||
the list names it, the parts of a pack must add up to the download
|
||||
size the README prints, and nothing else may be attached. An upload that
|
||||
size the README prints, and nothing else may be attached. The record holds
|
||||
one pack for each platform the registry marks release_pack, and no other
|
||||
pack. An upload that
|
||||
failed in the loop, or a part re-uploaded from an older build, otherwise
|
||||
publishes a release that `sha256sum --check` rejects for every user.
|
||||
|
||||
@@ -29,6 +31,7 @@ from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from build_port import PORT_ZIP
|
||||
from check_release_assets import digests_of
|
||||
from common import list_release_pack_platforms
|
||||
from largefiles import LARGE_FILES_REPO
|
||||
from release_record import RECORD, load_record
|
||||
|
||||
@@ -69,15 +72,31 @@ def compare_release(
|
||||
assets: dict[str, Asset],
|
||||
tag: str,
|
||||
local_files: dict[str, str] | None = None,
|
||||
released: set[str] | None = None,
|
||||
) -> list[Finding]:
|
||||
"""Every way the release can differ from what was built.
|
||||
|
||||
*local_files* maps a file kept beside the packs (the checksum list, its
|
||||
signature) to its local sha256: the copy on the release must be that one.
|
||||
*released* names the platforms the registry marks release_pack: each one
|
||||
has a recorded pack, and no recorded pack serves only other platforms.
|
||||
"""
|
||||
findings: list[Finding] = []
|
||||
if record.get("tag") != tag:
|
||||
findings.append(Finding("tag", tag, f"release.json records {record.get('tag')}"))
|
||||
if released is not None:
|
||||
packs = record.get("packs", {})
|
||||
served = {p for pack in packs.values() for p in pack.get("platforms", ())}
|
||||
for platform in sorted(released - served):
|
||||
findings.append(Finding(
|
||||
"no-pack", platform, "release_pack: true, no pack in release.json"
|
||||
))
|
||||
for pack_name, pack in sorted(packs.items()):
|
||||
if not set(pack.get("platforms", ())) & released:
|
||||
findings.append(Finding(
|
||||
"not-released", pack_name,
|
||||
"serves no platform marked release_pack in platforms/_registry.yml",
|
||||
))
|
||||
|
||||
recorded = {
|
||||
name: pack_name
|
||||
@@ -167,6 +186,10 @@ def main() -> int:
|
||||
help=f"local {SUMS} the packs were listed in. Its copy and signature on "
|
||||
f"the release must be these files (default: the list attached to the release)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platforms-dir", default="platforms",
|
||||
help="registry whose release_pack platforms each need a pack (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -199,7 +222,8 @@ def main() -> int:
|
||||
print(f"ERROR: {SUMS}: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
findings = compare_release(record, sums, assets, tag, local_files)
|
||||
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
|
||||
findings = compare_release(record, sums, assets, tag, local_files, released)
|
||||
if args.json:
|
||||
print(json.dumps(
|
||||
{"tag": tag, "draft": draft, "assets": len(assets),
|
||||
|
||||
@@ -437,6 +437,25 @@ def list_registered_platforms(
|
||||
return platforms
|
||||
|
||||
|
||||
def list_release_pack_platforms(
|
||||
platforms_dir: str = "platforms",
|
||||
include_archived: bool = False,
|
||||
) -> list[str]:
|
||||
"""Registered platforms a release publishes a ZIP pack for.
|
||||
|
||||
`release_pack: true` in _registry.yml is the whole list: the build of
|
||||
`--all`, the pack checks, the release record and every download table
|
||||
read it. The other platforms keep their install manifest and are served
|
||||
by the installer.
|
||||
"""
|
||||
registry = load_platform_registry(platforms_dir)
|
||||
return [
|
||||
name
|
||||
for name in list_registered_platforms(platforms_dir, include_archived)
|
||||
if registry.get(name, {}).get("release_pack") is True
|
||||
]
|
||||
|
||||
|
||||
# A list this short is printed whole rather than pointed at.
|
||||
SHORT_LIST = 12
|
||||
|
||||
|
||||
+29
-14
@@ -57,6 +57,7 @@ from common import (
|
||||
canonical_target_name,
|
||||
platform_files_target,
|
||||
list_registered_platforms,
|
||||
list_release_pack_platforms,
|
||||
list_system_ids,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
@@ -2623,7 +2624,9 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
if args.platform:
|
||||
platforms = [args.platform]
|
||||
elif args.all:
|
||||
platforms = list_registered_platforms(
|
||||
# The packs --all builds: a platform without release_pack has no pack
|
||||
# to check, and its absence from the directory is no failure.
|
||||
platforms = list_release_pack_platforms(
|
||||
args.platforms_dir, include_archived=args.include_archived
|
||||
)
|
||||
else:
|
||||
@@ -3112,7 +3115,9 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
help="Platform name (e.g., retroarch)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all", action="store_true", help="Generate packs for all active platforms"
|
||||
"--all", action="store_true",
|
||||
help="Generate the release packs (release_pack in the registry), or with"
|
||||
" --manifest the install manifests of every platform",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
|
||||
@@ -3264,10 +3269,14 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
)
|
||||
return
|
||||
if args.list:
|
||||
# Archived platforms build like the others but have no release pack.
|
||||
# Every platform builds with --platform. --all builds the release
|
||||
# packs only, the others are served by the installer.
|
||||
active = set(list_registered_platforms(args.platforms_dir))
|
||||
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
|
||||
for p in list_platforms(args.platforms_dir):
|
||||
print(p if p in active else f"{p} (archived)")
|
||||
marks = [m for m, on in (("archived", p not in active),
|
||||
("installer only", p not in released)) if on]
|
||||
print(f"{p} ({', '.join(marks)})" if marks else p)
|
||||
return
|
||||
if args.list_emulators:
|
||||
list_emulator_profiles(args.emulators_dir)
|
||||
@@ -3393,15 +3402,20 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
system_filter = getattr(args, "system_ids", None)
|
||||
|
||||
# Platform mode
|
||||
if args.all:
|
||||
# Install manifests cover every registered platform. "Archived" means
|
||||
if args.all and args.manifest:
|
||||
# Install manifests cover every registered platform: the installer
|
||||
# serves each one, with or without a release pack. "Archived" means
|
||||
# upstream is no longer scraped on a schedule, not that the platform
|
||||
# lost its users: its packs still ship, so the installer still needs
|
||||
# its file list.
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir,
|
||||
include_archived=args.include_archived or args.manifest,
|
||||
# lost its users.
|
||||
platforms = list_registered_platforms(args.platforms_dir, include_archived=True)
|
||||
elif args.all:
|
||||
# A release publishes the packs the registry marks release_pack and
|
||||
# nothing else: a pack built for another platform is one nobody uploads.
|
||||
platforms = list_release_pack_platforms(
|
||||
args.platforms_dir, include_archived=args.include_archived
|
||||
)
|
||||
if not platforms:
|
||||
parser.error("no registered platform has release_pack: true")
|
||||
elif args.platform:
|
||||
platforms = [args.platform]
|
||||
else:
|
||||
@@ -4789,10 +4803,11 @@ def _expected_pack_names(
|
||||
}
|
||||
for p in platforms
|
||||
}
|
||||
# Groups depend on which platforms a run built: --all alone or with
|
||||
# --include-archived. Both compositions are names a pack can carry.
|
||||
# Groups depend on which platforms a run built: the release packs of
|
||||
# --all, alone or with --include-archived. Both compositions are names a
|
||||
# pack can carry.
|
||||
for include_archived in (False, True):
|
||||
pool = list_registered_platforms(
|
||||
pool = list_release_pack_platforms(
|
||||
platforms_dir, include_archived=include_archived
|
||||
)
|
||||
cache = None
|
||||
|
||||
+27
-3
@@ -317,8 +317,27 @@ def _match_key(label: str) -> set[str]:
|
||||
} - {""}
|
||||
|
||||
|
||||
def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
"""Verify that check counts match between verify and pack for each platform."""
|
||||
def release_pack_keys(platforms_dir: str = "platforms", include_archived: bool = False) -> set[str]:
|
||||
"""Comparable identities of the platforms --all builds a pack for: their
|
||||
registry ids and display names, normalized as _match_key does."""
|
||||
from common import list_release_pack_platforms, load_platform_config # noqa: PLC0415
|
||||
|
||||
keys: set[str] = set()
|
||||
for name in list_release_pack_platforms(platforms_dir, include_archived):
|
||||
keys |= _match_key(name)
|
||||
keys |= _match_key(str(load_platform_config(name, platforms_dir).get("platform", name)))
|
||||
return keys
|
||||
|
||||
|
||||
def check_consistency(
|
||||
verify_output: str, pack_output: str, released: set[str] | None = None,
|
||||
) -> bool:
|
||||
"""Verify that check counts match between verify and pack for each platform.
|
||||
|
||||
*released* holds the identities of the platforms with a release pack
|
||||
(release_pack_keys). verify judges every platform, the build only those:
|
||||
a platform outside it has no pack to compare, which is no failure.
|
||||
"""
|
||||
v = parse_verify_counts(verify_output)
|
||||
p = parse_pack_counts(pack_output)
|
||||
excluded = parse_pack_exclusions(pack_output)
|
||||
@@ -377,6 +396,8 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
f"({p_excluded} unsafe excluded) != verify {v_ok}/{v_total}"
|
||||
)
|
||||
all_ok = False
|
||||
elif released is not None and not (v_names & released):
|
||||
print(f" {v_label}: {v_ok}/{v_total} (no release pack, served by the installer)")
|
||||
else:
|
||||
print(f" {v_label}: {v_ok}/{v_total} (no separate pack)")
|
||||
all_ok = False
|
||||
@@ -1023,7 +1044,10 @@ def main() -> None:
|
||||
skip_notice("consistency", "no full pack built")
|
||||
results["consistency"] = SKIPPED
|
||||
elif pack_output and verify_output:
|
||||
ok = check_consistency(verify_output, pack_output)
|
||||
ok = check_consistency(
|
||||
verify_output, pack_output,
|
||||
release_pack_keys(include_archived=args.include_archived),
|
||||
)
|
||||
# Counts can agree while the paths differ, both ways at once.
|
||||
ok = check_pack_destinations(destinations_file, pack_output) and ok
|
||||
ok = check_emulator_packs() and ok
|
||||
|
||||
@@ -26,11 +26,15 @@ import split_pack
|
||||
from common import (
|
||||
ArtifactLockBusy,
|
||||
artifact_lock,
|
||||
list_registered_platforms,
|
||||
list_release_pack_platforms,
|
||||
write_text_atomic,
|
||||
)
|
||||
|
||||
RECORD = "release.json"
|
||||
# Each registered platform has a page at get/<platform>/ on the site: the
|
||||
# current pack's files, or the command when no release publishes one. The
|
||||
# address outlives every release, so a link written elsewhere keeps working.
|
||||
GET_DIR = "get"
|
||||
|
||||
|
||||
class UnfinishedSplitError(Exception):
|
||||
@@ -46,11 +50,12 @@ def release_owners(platforms_dir: str = "platforms") -> dict[str, list[str]]:
|
||||
|
||||
Rebuilt from the name builder, never matched by substring: matched by
|
||||
substring, RetroArch_v1.22.2_AndroidArm64V8A_BIOS_Pack.zip sorts before
|
||||
the full pack and was taken for the RetroArch pack.
|
||||
the full pack and was taken for the RetroArch pack. Only the platforms
|
||||
marked release_pack own a release pack.
|
||||
"""
|
||||
import generate_pack
|
||||
|
||||
platforms = list_registered_platforms(platforms_dir, include_archived=True)
|
||||
platforms = list_release_pack_platforms(platforms_dir, include_archived=True)
|
||||
owners: dict[str, list[str]] = {}
|
||||
for platform, names in generate_pack._expected_pack_names(
|
||||
platforms, platforms_dir, None, None
|
||||
@@ -127,6 +132,35 @@ def load_record(path: str | Path = RECORD) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
def get_page(platform: str, site: str = "") -> str:
|
||||
"""The permanent download address of a platform, under *site*."""
|
||||
return f"{site}{GET_DIR}/{platform}/"
|
||||
|
||||
|
||||
def advertised(record: dict, platforms_dir: str = "platforms") -> dict:
|
||||
"""The record as the download tables show it: release_pack platforms only.
|
||||
|
||||
A release cut before a platform lost its release_pack still holds that
|
||||
platform's pack. The tables follow the registry, so a platform left out
|
||||
is offered the command, and a pack two platforms shared names the one
|
||||
still released.
|
||||
"""
|
||||
if not record:
|
||||
return {}
|
||||
released = set(list_release_pack_platforms(platforms_dir, include_archived=True))
|
||||
packs = {}
|
||||
for name, pack in record.get("packs", {}).items():
|
||||
serves = [p for p in pack.get("platforms", ()) if p in released]
|
||||
if serves:
|
||||
packs[name] = {**pack, "platforms": serves}
|
||||
return {**record, "packs": packs}
|
||||
|
||||
|
||||
def load_advertised(path: str | Path = RECORD, platforms_dir: str = "platforms") -> dict:
|
||||
"""The committed record, reduced to the platforms a release publishes."""
|
||||
return advertised(load_record(path), platforms_dir)
|
||||
|
||||
|
||||
def pack_name_for(platform: str, record: dict) -> str | None:
|
||||
"""Name of the release pack that serves a platform, if it has one."""
|
||||
for name, pack in sorted(record.get("packs", {}).items()):
|
||||
|
||||
@@ -391,6 +391,31 @@ def _save_folder_once_paths(path: Path, document: dict) -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def _registry_contract() -> list[str]:
|
||||
"""The registry against its schema, and a release that still ships a pack.
|
||||
|
||||
`release_pack` is the only list of the ZIPs a release publishes: a
|
||||
registry where no active platform carries it would cut a release with
|
||||
no pack at all, and every download link would lead to the command.
|
||||
"""
|
||||
registry_path = ROOT / "platforms" / "_registry.yml"
|
||||
if not registry_path.is_file():
|
||||
return []
|
||||
with registry_path.open(encoding="utf-8") as handle:
|
||||
registry = yaml_load(handle) or {}
|
||||
label = str(registry_path.relative_to(ROOT))
|
||||
out = _errors(_validator("registry.schema.json"), registry, label)
|
||||
platforms = registry.get("platforms") or {}
|
||||
if isinstance(platforms, dict) and not any(
|
||||
isinstance(entry, dict)
|
||||
and entry.get("release_pack") is True
|
||||
and entry.get("status", "active") == "active"
|
||||
for entry in platforms.values()
|
||||
):
|
||||
out.append(f"{label}: no active platform has release_pack: true")
|
||||
return out
|
||||
|
||||
|
||||
def _save_directory_blocks() -> list[str]:
|
||||
"""Each registry save_directory is null or a folder and a by_core flag.
|
||||
|
||||
@@ -572,6 +597,7 @@ def main() -> int:
|
||||
errors.extend(_unknown_data_dir_refs())
|
||||
errors.extend(_repository_layout_in_profiles())
|
||||
errors.extend(_save_directory_blocks())
|
||||
errors.extend(_registry_contract())
|
||||
|
||||
if not args.source_only:
|
||||
database_path = ROOT / "database.json"
|
||||
|
||||
@@ -346,7 +346,8 @@ class ListPlatformsTakesTheSharedFlag(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
lines = proc.stdout.splitlines()
|
||||
self.assertIn("retropie (archived)", lines)
|
||||
self.assertIn("retropie (archived, installer only)", lines)
|
||||
self.assertIn("knulli (installer only)", lines)
|
||||
self.assertIn("retroarch", lines)
|
||||
|
||||
|
||||
|
||||
@@ -140,12 +140,19 @@ class VerifyPackChoice(unittest.TestCase):
|
||||
self._run(self._args())
|
||||
self.assertEqual(out.getvalue().strip().splitlines()[-1], "Verified 1 pack: all OK")
|
||||
|
||||
def test_include_archived_reaches_the_archived_pack(self):
|
||||
def test_the_sweep_checks_release_packs_and_a_named_platform_its_own(self):
|
||||
"""--all checks the packs a release publishes (release_pack): the
|
||||
RetroPie pack, archived and served by the installer, is left to a
|
||||
check that names it."""
|
||||
stem = self.gp._platform_pack_stem(["retropie"], "retropie", "platforms")
|
||||
self._touch(f"{stem}_BIOS_Pack.zip")
|
||||
self._touch(f"{self.stem}_BIOS_Pack.zip")
|
||||
checked = self._run(
|
||||
self._args(platform=None, all=True, include_archived=True)
|
||||
)
|
||||
self.assertIn(("retroarch", f"{self.stem}_BIOS_Pack.zip"), checked)
|
||||
self.assertNotIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked)
|
||||
checked = self._run(self._args(platform="retropie"))
|
||||
self.assertIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user