From 7ba899d4675c3ffbce6411f21f8b09e4a88b6473 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:07:54 +0100 Subject: [PATCH] feat: build the installer as a PyPI package scripts/build_pypi.py writes a wheel and an sdist from install.py, the copy install.sh pins, and packaging/pypi. The version is the date of the release in release.json. validate.yml now runs on packaging/ changes. --- .github/workflows/validate.yml | 3 + packaging/pypi/README.md | 22 ++++ packaging/pypi/pyproject.toml | 37 ++++++ scripts/build_pypi.py | 212 +++++++++++++++++++++++++++++++++ tests/test_pypi_package.py | 200 +++++++++++++++++++++++++++++++ wiki/release-process.md | 15 ++- 6 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 packaging/pypi/README.md create mode 100644 packaging/pypi/pyproject.toml create mode 100644 scripts/build_pypi.py create mode 100644 tests/test_pypi_package.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8a75a5dee..06853c125 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -19,6 +19,8 @@ on: - "install.ps1" # The handheld port bundles install.py and its tests read both. - "ports/**" + # The PyPI package is built from install.py and its tests build it. + - "packaging/**" # Every input of the site build, so a page that would fail the deploy # fails here first (tests/test_workflow_paths.py). - "wiki/**" @@ -50,6 +52,7 @@ on: - "install.sh" - "install.ps1" - "ports/**" + - "packaging/**" - "wiki/**" - "docs_assets/**" - "provenance/**" diff --git a/packaging/pypi/README.md b/packaging/pypi/README.md new file mode 100644 index 000000000..98f85de7a --- /dev/null +++ b/packaging/pypi/README.md @@ -0,0 +1,22 @@ +# RetroBIOS + +Puts the BIOS files your emulators need in the right folder. + +It finds your setup and its BIOS folder, downloads only the missing files and +checks each one. It works with RetroArch, Batocera, EmuDeck, RetroBat, +Recalbox, RetroDECK, RomM and more. + +Run it: + +```bash +uvx retrobios +``` + +Or `pipx run retrobios`, or `pip install retrobios` and then `retrobios`. +Python 3.8 or newer. + +Add `--check` to see what is missing without changing anything, and `--help` +for every option. + +Guides, the list of setups and the files each one needs: +https://abdess.github.io/retrobios/ diff --git a/packaging/pypi/pyproject.toml b/packaging/pypi/pyproject.toml new file mode 100644 index 000000000..eeb1d92f4 --- /dev/null +++ b/packaging/pypi/pyproject.toml @@ -0,0 +1,37 @@ +# The package PyPI serves as `retrobios`: install.py and the command that +# runs it, nothing else. The root pyproject.toml is the contributors' +# environment. scripts/build_pypi.py writes the wheel and the sdist from this +# file, README.md, LICENSE and install.py, and writes retrobios/__init__.py +# with the version. This file is the one a build from the sdist reads. +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "retrobios" +dynamic = ["version"] +description = "Install the BIOS files your emulators need" +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +license-files = ["LICENSE"] +dependencies = [] +classifiers = [ + "Environment :: Console", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: System :: Emulators", +] + +[project.urls] +Homepage = "https://abdess.github.io/retrobios/" +Source = "https://github.com/Abdess/retrobios" + +[project.scripts] +retrobios = "retrobios:main" + +[tool.setuptools] +packages = ["retrobios"] + +[tool.setuptools.dynamic] +version = { attr = "retrobios.__version__" } diff --git a/scripts/build_pypi.py b/scripts/build_pypi.py new file mode 100644 index 000000000..efbeca92a --- /dev/null +++ b/scripts/build_pypi.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Build the `retrobios` package PyPI serves: install.py as a command. + +The wheel and the sdist hold install.py, byte for byte the copy the +bootstraps pin, a two-line launcher that runs it as a script, the metadata, +README.md and LICENSE. Nothing of the collection. The metadata comes from +packaging/pypi/pyproject.toml. + +The version is the date of the release in release.json: v2026.10.10 gives +2026.10.10. A release writes release.json first, so the package published +with it carries its date. + +Both archives are written here with the standard library rather than by a +build backend: the bytes then depend on the inputs alone, on any machine and +with no network. The sdist still names setuptools as its backend, for pip +and uv to build it. + +Usage: + python scripts/build_pypi.py + python scripts/build_pypi.py --output-dir tmp/pypi-dist +""" + +from __future__ import annotations + +import argparse +import base64 +import calendar +import gzip +import hashlib +import io +import json +import re +import tarfile +import zipfile +from pathlib import Path + +from sitebootstrap import PIN, SH_NAME + +ROOT = Path(__file__).resolve().parent.parent +PACKAGE_DIR = ROOT / "packaging" / "pypi" +DEFAULT_OUTPUT = ROOT / "tmp" / "pypi-dist" +MODULE = "retrobios" + +# The launcher. runpy runs install.py as __main__, so its own __main__ block +# keeps the exit codes it gives a reader that left (141) and Ctrl+C (130). +LAUNCHER = '''"""The RetroBIOS installer, install.py, run as the retrobios command.""" + +import runpy +from pathlib import Path + +__version__ = "{version}" + + +def main() -> None: + runpy.run_path(str(Path(__file__).with_name("install.py")), run_name="__main__") +''' + +# The [project] keys the metadata below writes. Another key in pyproject.toml +# would reach a build from the sdist and not the wheel written here. +PROJECT_KEYS = frozenset({ + "name", "dynamic", "description", "readme", "requires-python", "license", + "license-files", "dependencies", "classifiers", "urls", "scripts", +}) + + +def package_version(release: dict) -> str: + """The release date as a version, without the zeros PEP 440 drops.""" + tag = str(release.get("tag", "")) + match = re.fullmatch(r"v(\d{4})\.(\d{2})\.(\d{2})", tag) + if match is None: + raise ValueError(f"release.json names no dated release tag: {tag!r}") + return ".".join(str(int(part)) for part in match.groups()) + + +def load_project() -> dict: + import tomllib # noqa: PLC0415 # 3.11, only where a package is built + + with (PACKAGE_DIR / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + unknown = set(project) - PROJECT_KEYS + if unknown: + raise ValueError(f"build_pypi.py writes no metadata for {sorted(unknown)}") + if project["dependencies"] or project["dynamic"] != ["version"]: + raise ValueError("the package has no dependencies and only its version is dynamic") + return project + + +def metadata(project: dict, version: str, readme: str) -> str: + """Core metadata 2.4, as the wheel's METADATA and the sdist's PKG-INFO.""" + lines = [ + "Metadata-Version: 2.4", + f"Name: {project['name']}", + f"Version: {version}", + f"Summary: {project['description']}", + *(f"Project-URL: {label}, {url}" for label, url in project["urls"].items()), + f"License-Expression: {project['license']}", + *(f"License-File: {name}" for name in project["license-files"]), + *(f"Classifier: {name}" for name in project["classifiers"]), + f"Requires-Python: {project['requires-python']}", + "Description-Content-Type: text/markdown", + ] + return "\n".join(lines) + "\n\n" + readme + + +def installer_bytes() -> bytes: + """install.py, refused unless it is the one install.sh pins: the package + then runs the installer the one-line command runs, under the same id.""" + data = (ROOT / "install.py").read_bytes() + pins = PIN[SH_NAME].findall((ROOT / "install.sh").read_text(encoding="utf-8")) + if pins != [hashlib.sha256(data).hexdigest()]: + raise ValueError("install.py is not the installer install.sh pins: recompute the pin") + return data + + +def package_files(version: str) -> tuple[dict, dict[str, bytes]]: + """The project table, and the files both archives ship, by path.""" + project = load_project() + readme = (PACKAGE_DIR / project["readme"]).read_text(encoding="utf-8") + files = { + f"{MODULE}/__init__.py": LAUNCHER.format(version=version).encode(), + f"{MODULE}/install.py": installer_bytes(), + "METADATA": metadata(project, version, readme).encode("utf-8"), + "README.md": readme.encode("utf-8"), + "pyproject.toml": (PACKAGE_DIR / "pyproject.toml").read_bytes(), + } + for name in project["license-files"]: + files[name] = (ROOT / name).read_bytes() + return project, files + + +def _record_line(path: str, data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + return f"{path},sha256={digest},{len(data)}" + + +def write_wheel(path: Path, project: dict, files: dict[str, bytes], version: str, + stamp: tuple[int, int, int]) -> None: + info = f"{project['name']}-{version}.dist-info" + scripts = "".join(f"{name} = {target}\n" for name, target in project["scripts"].items()) + members = { + f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], + f"{MODULE}/install.py": files[f"{MODULE}/install.py"], + f"{info}/METADATA": files["METADATA"], + f"{info}/WHEEL": (b"Wheel-Version: 1.0\nGenerator: retrobios build_pypi.py\n" + b"Root-Is-Purelib: true\nTag: py3-none-any\n"), + f"{info}/entry_points.txt": f"[console_scripts]\n{scripts}".encode(), + **{f"{info}/licenses/{name}": files[name] for name in project["license-files"]}, + } + record = "".join(_record_line(name, data) + "\n" for name, data in members.items()) + members[f"{info}/RECORD"] = (record + f"{info}/RECORD,,\n").encode() + with zipfile.ZipFile(path, "w") as archive: + for name, data in members.items(): + entry = zipfile.ZipInfo(name, date_time=(*stamp, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + archive.writestr(entry, data) + + +def write_sdist(path: Path, project: dict, files: dict[str, bytes], version: str, + stamp: tuple[int, int, int]) -> None: + top = f"{project['name']}-{version}" + members = { + "PKG-INFO": files["METADATA"], + "pyproject.toml": files["pyproject.toml"], + "README.md": files["README.md"], + **{name: files[name] for name in project["license-files"]}, + f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], + f"{MODULE}/install.py": files[f"{MODULE}/install.py"], + } + mtime = calendar.timegm((*stamp, 0, 0, 0)) + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as archive: + for name, data in members.items(): + entry = tarfile.TarInfo(f"{top}/{name}") + entry.size, entry.mtime, entry.mode = len(data), mtime, 0o644 + archive.addfile(entry, io.BytesIO(data)) + with path.open("wb") as handle, gzip.GzipFile(fileobj=handle, mode="wb", filename="", + mtime=mtime) as packed: + packed.write(raw.getvalue()) + + +def build(output_dir: Path) -> tuple[Path, Path]: + """The wheel and the sdist in output_dir, replacing earlier builds there.""" + version = package_version(json.loads((ROOT / "release.json").read_text(encoding="utf-8"))) + year, month, day = (int(part) for part in version.split(".")) + stamp = (year, month, day) + project, files = package_files(version) + output_dir.mkdir(parents=True, exist_ok=True) + for pattern in ("*.whl", "*.tar.gz"): + for old in output_dir.glob(f"{project['name']}-{pattern}"): + old.unlink() + wheel = output_dir / f"{project['name']}-{version}-py3-none-any.whl" + sdist = output_dir / f"{project['name']}-{version}.tar.gz" + write_wheel(wheel, project, files, version, stamp) + write_sdist(sdist, project, files, version, stamp) + return wheel, sdist + + +def main() -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0]) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT, + help="where the wheel and the sdist go (default: tmp/pypi-dist)") + args = parser.parse_args() + for path in build(args.output_dir): + print(path) + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/tests/test_pypi_package.py b/tests/test_pypi_package.py new file mode 100644 index 000000000..cf3e7bdb5 --- /dev/null +++ b/tests/test_pypi_package.py @@ -0,0 +1,200 @@ +"""The `retrobios` package on PyPI is install.py and the command that runs it. + +`uvx retrobios`, `pipx run retrobios` and `pip install retrobios` reach the +installer the one-line command runs: the same bytes, so the same id, the +same file lists, cache and exit codes. scripts/build_pypi.py writes the +wheel and the sdist, and these tests read what it wrote and run it. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import email.parser +import importlib.util +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +import venv +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import build_pypi # noqa: E402 + +MODULE_FILES = {"retrobios/__init__.py", "retrobios/install.py"} + + +class Built(unittest.TestCase): + """One build per class, in a scratch folder.""" + + _scratch: tempfile.TemporaryDirectory + scratch: Path + wheel: Path + sdist: Path + version: str + + @classmethod + def setUpClass(cls) -> None: + cls._scratch = tempfile.TemporaryDirectory(prefix="pypi-") + cls.scratch = Path(cls._scratch.name) + cls.wheel, cls.sdist = build_pypi.build(cls.scratch / "dist") + cls.version = cls.wheel.name.split("-")[1] + + @classmethod + def tearDownClass(cls) -> None: + cls._scratch.cleanup() + + +class TheArchivesHoldTheInstallerAndNothingElse(Built): + + def test_wheel_members(self): + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(self.wheel) as archive: + names = set(archive.namelist()) + self.assertEqual(names, MODULE_FILES | { + f"{info}/{name}" for name in + ("METADATA", "WHEEL", "entry_points.txt", "RECORD", "licenses/LICENSE")}) + self.assertEqual(archive.read("retrobios/install.py"), + (REPO_ROOT / "install.py").read_bytes()) + entry_points = archive.read(f"{info}/entry_points.txt").decode() + meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + self.assertEqual(entry_points, "[console_scripts]\nretrobios = retrobios:main\n") + self.assertEqual(meta["Name"], "retrobios") + self.assertEqual(meta["Version"], self.version) + self.assertEqual(meta["Requires-Python"], ">=3.8") + self.assertIsNone(meta["Requires-Dist"]) + self.assertEqual(meta.get_payload(), + (REPO_ROOT / "packaging/pypi/README.md").read_text(encoding="utf-8")) + + def test_sdist_members(self): + top = f"retrobios-{self.version}" + with tarfile.open(self.sdist) as archive: + names = {member.name for member in archive.getmembers()} + installer = archive.extractfile(f"{top}/retrobios/install.py").read() + self.assertEqual(names, {f"{top}/{name}" for name in ( + *MODULE_FILES, "PKG-INFO", "pyproject.toml", "README.md", "LICENSE")}) + self.assertEqual(installer, (REPO_ROOT / "install.py").read_bytes()) + + def test_no_collection_file(self): + with zipfile.ZipFile(self.wheel) as archive: + names = archive.namelist() + with tarfile.open(self.sdist) as archive: + names += [name.split("/", 1)[1] for name in archive.getnames()] + for top in ("bios/", "data/", "install/", "platforms/", "emulators/"): + self.assertEqual([name for name in names if name.startswith(top)], [], top) + + def test_build_is_reproducible(self): + again = build_pypi.build(self.scratch / "again") + self.assertEqual([path.read_bytes() for path in again], + [self.wheel.read_bytes(), self.sdist.read_bytes()]) + + +class TheVersionIsTheReleaseDate(unittest.TestCase): + + def test_dated_tag(self): + self.assertEqual(build_pypi.package_version({"tag": "v2026.10.04"}), "2026.10.4") + + def test_other_tag_refused(self): + for tag in ("large-files", "2026.10.04", "v2026.10"): + with self.assertRaises(ValueError): + build_pypi.package_version({"tag": tag}) + + +def _python38() -> str | None: + found = shutil.which("python3.8") + if found is None and os.environ.get("CI"): + raise AssertionError("CI installs Python 3.8") + return found + + +class TheInstalledCommandIsTheInstaller(Built): + """Installed in a fresh venv, `retrobios` answers as `python install.py`.""" + + def _installed(self, python: str, name: str) -> Path: + env = self.scratch / name + made = subprocess.run([python, "-m", "venv", str(env)], capture_output=True, text=True, + check=False) + self.assertEqual(made.returncode, 0, made.stderr[-2000:]) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + done = subprocess.run( + [str(bindir / "python"), "-m", "pip", "install", "--no-index", "--no-deps", + "--disable-pip-version-check", str(self.wheel)], + capture_output=True, text=True, check=False) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + return bindir / "retrobios" + + def _same_answers(self, python: str, name: str) -> None: + command = self._installed(python, name) + for args in (["--help"], ["--version"], ["--no-such-option"]): + with self.subTest(python=name, args=args): + packaged = subprocess.run([str(command), *args], capture_output=True, + text=True, check=False, cwd=self.scratch) + script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], + capture_output=True, text=True, check=False, + cwd=self.scratch) + self.assertEqual( + (packaged.returncode, packaged.stdout, packaged.stderr), + (script.returncode, script.stdout, script.stderr)) + + def test_this_python(self): + self._same_answers(sys.executable, "venv-current") + + def test_python_38(self): + python = _python38() + if python is None: + self.skipTest("python3.8 is not on PATH") + self._same_answers(python, "venv-38") + + +def _setuptools_builds_the_sdist() -> bool: + """setuptools 77 reads the license expression the sdist declares.""" + if importlib.util.find_spec("setuptools") is None: + return False + import setuptools # noqa: PLC0415 + + return int(setuptools.__version__.split(".")[0]) >= 77 + + +class TheSdistBuildsTheSameWheel(Built): + """pip and uv build the sdist with setuptools: the files must be those the + wheel written here holds.""" + + def test_setuptools_wheel(self): + if not _setuptools_builds_the_sdist(): + if os.environ.get("CI"): + self.fail("CI installs setuptools 77 or newer") + self.skipTest("setuptools 77 or newer is not installed") + out = self.scratch / "from-sdist" + work = self.scratch / "unpacked" + with tarfile.open(self.sdist) as archive: + archive.extractall(work, filter="data") + done = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-build-isolation", "--no-deps", + "--no-index", "--disable-pip-version-check", "-w", str(out), + str(work / f"retrobios-{self.version}")], + capture_output=True, text=True, check=False) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + (built,) = out.glob("*.whl") + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(built) as theirs, zipfile.ZipFile(self.wheel) as ours: + self.assertEqual(set(theirs.namelist()) - {f"{info}/top_level.txt"}, + set(ours.namelist())) + for name in (*MODULE_FILES, f"{info}/entry_points.txt"): + self.assertEqual(theirs.read(name), ours.read(name), name) + fields = ("Name", "Version", "Summary", "License-Expression", "Requires-Python") + theirs_meta, ours_meta = ( + email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + for archive in (theirs, ours)) + for field in fields: + self.assertEqual(theirs_meta[field], ours_meta[field], field) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/release-process.md b/wiki/release-process.md index b0367098b..71220c265 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -110,8 +110,8 @@ scripts stay Python. `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh` or `install.ps1` (the bootstraps pin the installer's SHA-256, so editing one without the other fails the run), -`ports/**` (the handheld port bundles the installer), or any input of the site -build. The path lists are spelled out once per event because the workflow +`ports/**` (the handheld port bundles the installer), `packaging/**` (the PyPI +package is built from it), or any input of the site build. The path lists are spelled out once per event because the workflow parser reads no YAML anchor, and a test holds the two equal. **Concurrency.** Per-PR group on a pull request, per-ref on a push. A pull @@ -312,6 +312,17 @@ git status --short git add release.json README.md install/*.json install/targets/*.json git commit -m "chore: record release v${DATE}" && git push +# 5b. The PyPI package, `uvx retrobios`: install.py and its command. The +# version is the date of the tag release.json now names (v2026.10.04 +# gives 2026.10.4), so a release is what bumps it. The build replaces +# any earlier one in tmp/pypi-dist. Build, run the wheel, then upload +# the two files. The token stays in its file: never in the +# repository, a script or a log. +python scripts/build_pypi.py --output-dir tmp/pypi-dist +python -m unittest tests.test_pypi_package +uvx --from tmp/pypi-dist/retrobios-*.whl retrobios --help +uv publish --token "$(cat ~/.config/retrobios/pypi-token)" tmp/pypi-dist/* + # 6. Keep only the new release plus large-files: an older pack carries hashes # the platforms no longer check, so it misleads more than it helps gh release list --json tagName,createdAt \