From 7ba899d4675c3ffbce6411f21f8b09e4a88b6473 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:07:54 +0100 Subject: [PATCH 1/8] feat: build the installer as a PyPI package scripts/build_pypi.py writes a wheel and an sdist from install.py, the copy install.sh pins, and packaging/pypi. The version is the date of the release in release.json. validate.yml now runs on packaging/ changes. --- .github/workflows/validate.yml | 3 + packaging/pypi/README.md | 22 ++++ packaging/pypi/pyproject.toml | 37 ++++++ scripts/build_pypi.py | 212 +++++++++++++++++++++++++++++++++ tests/test_pypi_package.py | 200 +++++++++++++++++++++++++++++++ wiki/release-process.md | 15 ++- 6 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 packaging/pypi/README.md create mode 100644 packaging/pypi/pyproject.toml create mode 100644 scripts/build_pypi.py create mode 100644 tests/test_pypi_package.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8a75a5dee..06853c125 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -19,6 +19,8 @@ on: - "install.ps1" # The handheld port bundles install.py and its tests read both. - "ports/**" + # The PyPI package is built from install.py and its tests build it. + - "packaging/**" # Every input of the site build, so a page that would fail the deploy # fails here first (tests/test_workflow_paths.py). - "wiki/**" @@ -50,6 +52,7 @@ on: - "install.sh" - "install.ps1" - "ports/**" + - "packaging/**" - "wiki/**" - "docs_assets/**" - "provenance/**" diff --git a/packaging/pypi/README.md b/packaging/pypi/README.md new file mode 100644 index 000000000..98f85de7a --- /dev/null +++ b/packaging/pypi/README.md @@ -0,0 +1,22 @@ +# RetroBIOS + +Puts the BIOS files your emulators need in the right folder. + +It finds your setup and its BIOS folder, downloads only the missing files and +checks each one. It works with RetroArch, Batocera, EmuDeck, RetroBat, +Recalbox, RetroDECK, RomM and more. + +Run it: + +```bash +uvx retrobios +``` + +Or `pipx run retrobios`, or `pip install retrobios` and then `retrobios`. +Python 3.8 or newer. + +Add `--check` to see what is missing without changing anything, and `--help` +for every option. + +Guides, the list of setups and the files each one needs: +https://abdess.github.io/retrobios/ diff --git a/packaging/pypi/pyproject.toml b/packaging/pypi/pyproject.toml new file mode 100644 index 000000000..eeb1d92f4 --- /dev/null +++ b/packaging/pypi/pyproject.toml @@ -0,0 +1,37 @@ +# The package PyPI serves as `retrobios`: install.py and the command that +# runs it, nothing else. The root pyproject.toml is the contributors' +# environment. scripts/build_pypi.py writes the wheel and the sdist from this +# file, README.md, LICENSE and install.py, and writes retrobios/__init__.py +# with the version. This file is the one a build from the sdist reads. +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "retrobios" +dynamic = ["version"] +description = "Install the BIOS files your emulators need" +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +license-files = ["LICENSE"] +dependencies = [] +classifiers = [ + "Environment :: Console", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: System :: Emulators", +] + +[project.urls] +Homepage = "https://abdess.github.io/retrobios/" +Source = "https://github.com/Abdess/retrobios" + +[project.scripts] +retrobios = "retrobios:main" + +[tool.setuptools] +packages = ["retrobios"] + +[tool.setuptools.dynamic] +version = { attr = "retrobios.__version__" } diff --git a/scripts/build_pypi.py b/scripts/build_pypi.py new file mode 100644 index 000000000..efbeca92a --- /dev/null +++ b/scripts/build_pypi.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Build the `retrobios` package PyPI serves: install.py as a command. + +The wheel and the sdist hold install.py, byte for byte the copy the +bootstraps pin, a two-line launcher that runs it as a script, the metadata, +README.md and LICENSE. Nothing of the collection. The metadata comes from +packaging/pypi/pyproject.toml. + +The version is the date of the release in release.json: v2026.10.10 gives +2026.10.10. A release writes release.json first, so the package published +with it carries its date. + +Both archives are written here with the standard library rather than by a +build backend: the bytes then depend on the inputs alone, on any machine and +with no network. The sdist still names setuptools as its backend, for pip +and uv to build it. + +Usage: + python scripts/build_pypi.py + python scripts/build_pypi.py --output-dir tmp/pypi-dist +""" + +from __future__ import annotations + +import argparse +import base64 +import calendar +import gzip +import hashlib +import io +import json +import re +import tarfile +import zipfile +from pathlib import Path + +from sitebootstrap import PIN, SH_NAME + +ROOT = Path(__file__).resolve().parent.parent +PACKAGE_DIR = ROOT / "packaging" / "pypi" +DEFAULT_OUTPUT = ROOT / "tmp" / "pypi-dist" +MODULE = "retrobios" + +# The launcher. runpy runs install.py as __main__, so its own __main__ block +# keeps the exit codes it gives a reader that left (141) and Ctrl+C (130). +LAUNCHER = '''"""The RetroBIOS installer, install.py, run as the retrobios command.""" + +import runpy +from pathlib import Path + +__version__ = "{version}" + + +def main() -> None: + runpy.run_path(str(Path(__file__).with_name("install.py")), run_name="__main__") +''' + +# The [project] keys the metadata below writes. Another key in pyproject.toml +# would reach a build from the sdist and not the wheel written here. +PROJECT_KEYS = frozenset({ + "name", "dynamic", "description", "readme", "requires-python", "license", + "license-files", "dependencies", "classifiers", "urls", "scripts", +}) + + +def package_version(release: dict) -> str: + """The release date as a version, without the zeros PEP 440 drops.""" + tag = str(release.get("tag", "")) + match = re.fullmatch(r"v(\d{4})\.(\d{2})\.(\d{2})", tag) + if match is None: + raise ValueError(f"release.json names no dated release tag: {tag!r}") + return ".".join(str(int(part)) for part in match.groups()) + + +def load_project() -> dict: + import tomllib # noqa: PLC0415 # 3.11, only where a package is built + + with (PACKAGE_DIR / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + unknown = set(project) - PROJECT_KEYS + if unknown: + raise ValueError(f"build_pypi.py writes no metadata for {sorted(unknown)}") + if project["dependencies"] or project["dynamic"] != ["version"]: + raise ValueError("the package has no dependencies and only its version is dynamic") + return project + + +def metadata(project: dict, version: str, readme: str) -> str: + """Core metadata 2.4, as the wheel's METADATA and the sdist's PKG-INFO.""" + lines = [ + "Metadata-Version: 2.4", + f"Name: {project['name']}", + f"Version: {version}", + f"Summary: {project['description']}", + *(f"Project-URL: {label}, {url}" for label, url in project["urls"].items()), + f"License-Expression: {project['license']}", + *(f"License-File: {name}" for name in project["license-files"]), + *(f"Classifier: {name}" for name in project["classifiers"]), + f"Requires-Python: {project['requires-python']}", + "Description-Content-Type: text/markdown", + ] + return "\n".join(lines) + "\n\n" + readme + + +def installer_bytes() -> bytes: + """install.py, refused unless it is the one install.sh pins: the package + then runs the installer the one-line command runs, under the same id.""" + data = (ROOT / "install.py").read_bytes() + pins = PIN[SH_NAME].findall((ROOT / "install.sh").read_text(encoding="utf-8")) + if pins != [hashlib.sha256(data).hexdigest()]: + raise ValueError("install.py is not the installer install.sh pins: recompute the pin") + return data + + +def package_files(version: str) -> tuple[dict, dict[str, bytes]]: + """The project table, and the files both archives ship, by path.""" + project = load_project() + readme = (PACKAGE_DIR / project["readme"]).read_text(encoding="utf-8") + files = { + f"{MODULE}/__init__.py": LAUNCHER.format(version=version).encode(), + f"{MODULE}/install.py": installer_bytes(), + "METADATA": metadata(project, version, readme).encode("utf-8"), + "README.md": readme.encode("utf-8"), + "pyproject.toml": (PACKAGE_DIR / "pyproject.toml").read_bytes(), + } + for name in project["license-files"]: + files[name] = (ROOT / name).read_bytes() + return project, files + + +def _record_line(path: str, data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + return f"{path},sha256={digest},{len(data)}" + + +def write_wheel(path: Path, project: dict, files: dict[str, bytes], version: str, + stamp: tuple[int, int, int]) -> None: + info = f"{project['name']}-{version}.dist-info" + scripts = "".join(f"{name} = {target}\n" for name, target in project["scripts"].items()) + members = { + f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], + f"{MODULE}/install.py": files[f"{MODULE}/install.py"], + f"{info}/METADATA": files["METADATA"], + f"{info}/WHEEL": (b"Wheel-Version: 1.0\nGenerator: retrobios build_pypi.py\n" + b"Root-Is-Purelib: true\nTag: py3-none-any\n"), + f"{info}/entry_points.txt": f"[console_scripts]\n{scripts}".encode(), + **{f"{info}/licenses/{name}": files[name] for name in project["license-files"]}, + } + record = "".join(_record_line(name, data) + "\n" for name, data in members.items()) + members[f"{info}/RECORD"] = (record + f"{info}/RECORD,,\n").encode() + with zipfile.ZipFile(path, "w") as archive: + for name, data in members.items(): + entry = zipfile.ZipInfo(name, date_time=(*stamp, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + archive.writestr(entry, data) + + +def write_sdist(path: Path, project: dict, files: dict[str, bytes], version: str, + stamp: tuple[int, int, int]) -> None: + top = f"{project['name']}-{version}" + members = { + "PKG-INFO": files["METADATA"], + "pyproject.toml": files["pyproject.toml"], + "README.md": files["README.md"], + **{name: files[name] for name in project["license-files"]}, + f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], + f"{MODULE}/install.py": files[f"{MODULE}/install.py"], + } + mtime = calendar.timegm((*stamp, 0, 0, 0)) + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as archive: + for name, data in members.items(): + entry = tarfile.TarInfo(f"{top}/{name}") + entry.size, entry.mtime, entry.mode = len(data), mtime, 0o644 + archive.addfile(entry, io.BytesIO(data)) + with path.open("wb") as handle, gzip.GzipFile(fileobj=handle, mode="wb", filename="", + mtime=mtime) as packed: + packed.write(raw.getvalue()) + + +def build(output_dir: Path) -> tuple[Path, Path]: + """The wheel and the sdist in output_dir, replacing earlier builds there.""" + version = package_version(json.loads((ROOT / "release.json").read_text(encoding="utf-8"))) + year, month, day = (int(part) for part in version.split(".")) + stamp = (year, month, day) + project, files = package_files(version) + output_dir.mkdir(parents=True, exist_ok=True) + for pattern in ("*.whl", "*.tar.gz"): + for old in output_dir.glob(f"{project['name']}-{pattern}"): + old.unlink() + wheel = output_dir / f"{project['name']}-{version}-py3-none-any.whl" + sdist = output_dir / f"{project['name']}-{version}.tar.gz" + write_wheel(wheel, project, files, version, stamp) + write_sdist(sdist, project, files, version, stamp) + return wheel, sdist + + +def main() -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0]) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT, + help="where the wheel and the sdist go (default: tmp/pypi-dist)") + args = parser.parse_args() + for path in build(args.output_dir): + print(path) + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/tests/test_pypi_package.py b/tests/test_pypi_package.py new file mode 100644 index 000000000..cf3e7bdb5 --- /dev/null +++ b/tests/test_pypi_package.py @@ -0,0 +1,200 @@ +"""The `retrobios` package on PyPI is install.py and the command that runs it. + +`uvx retrobios`, `pipx run retrobios` and `pip install retrobios` reach the +installer the one-line command runs: the same bytes, so the same id, the +same file lists, cache and exit codes. scripts/build_pypi.py writes the +wheel and the sdist, and these tests read what it wrote and run it. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import email.parser +import importlib.util +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +import venv +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import build_pypi # noqa: E402 + +MODULE_FILES = {"retrobios/__init__.py", "retrobios/install.py"} + + +class Built(unittest.TestCase): + """One build per class, in a scratch folder.""" + + _scratch: tempfile.TemporaryDirectory + scratch: Path + wheel: Path + sdist: Path + version: str + + @classmethod + def setUpClass(cls) -> None: + cls._scratch = tempfile.TemporaryDirectory(prefix="pypi-") + cls.scratch = Path(cls._scratch.name) + cls.wheel, cls.sdist = build_pypi.build(cls.scratch / "dist") + cls.version = cls.wheel.name.split("-")[1] + + @classmethod + def tearDownClass(cls) -> None: + cls._scratch.cleanup() + + +class TheArchivesHoldTheInstallerAndNothingElse(Built): + + def test_wheel_members(self): + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(self.wheel) as archive: + names = set(archive.namelist()) + self.assertEqual(names, MODULE_FILES | { + f"{info}/{name}" for name in + ("METADATA", "WHEEL", "entry_points.txt", "RECORD", "licenses/LICENSE")}) + self.assertEqual(archive.read("retrobios/install.py"), + (REPO_ROOT / "install.py").read_bytes()) + entry_points = archive.read(f"{info}/entry_points.txt").decode() + meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + self.assertEqual(entry_points, "[console_scripts]\nretrobios = retrobios:main\n") + self.assertEqual(meta["Name"], "retrobios") + self.assertEqual(meta["Version"], self.version) + self.assertEqual(meta["Requires-Python"], ">=3.8") + self.assertIsNone(meta["Requires-Dist"]) + self.assertEqual(meta.get_payload(), + (REPO_ROOT / "packaging/pypi/README.md").read_text(encoding="utf-8")) + + def test_sdist_members(self): + top = f"retrobios-{self.version}" + with tarfile.open(self.sdist) as archive: + names = {member.name for member in archive.getmembers()} + installer = archive.extractfile(f"{top}/retrobios/install.py").read() + self.assertEqual(names, {f"{top}/{name}" for name in ( + *MODULE_FILES, "PKG-INFO", "pyproject.toml", "README.md", "LICENSE")}) + self.assertEqual(installer, (REPO_ROOT / "install.py").read_bytes()) + + def test_no_collection_file(self): + with zipfile.ZipFile(self.wheel) as archive: + names = archive.namelist() + with tarfile.open(self.sdist) as archive: + names += [name.split("/", 1)[1] for name in archive.getnames()] + for top in ("bios/", "data/", "install/", "platforms/", "emulators/"): + self.assertEqual([name for name in names if name.startswith(top)], [], top) + + def test_build_is_reproducible(self): + again = build_pypi.build(self.scratch / "again") + self.assertEqual([path.read_bytes() for path in again], + [self.wheel.read_bytes(), self.sdist.read_bytes()]) + + +class TheVersionIsTheReleaseDate(unittest.TestCase): + + def test_dated_tag(self): + self.assertEqual(build_pypi.package_version({"tag": "v2026.10.04"}), "2026.10.4") + + def test_other_tag_refused(self): + for tag in ("large-files", "2026.10.04", "v2026.10"): + with self.assertRaises(ValueError): + build_pypi.package_version({"tag": tag}) + + +def _python38() -> str | None: + found = shutil.which("python3.8") + if found is None and os.environ.get("CI"): + raise AssertionError("CI installs Python 3.8") + return found + + +class TheInstalledCommandIsTheInstaller(Built): + """Installed in a fresh venv, `retrobios` answers as `python install.py`.""" + + def _installed(self, python: str, name: str) -> Path: + env = self.scratch / name + made = subprocess.run([python, "-m", "venv", str(env)], capture_output=True, text=True, + check=False) + self.assertEqual(made.returncode, 0, made.stderr[-2000:]) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + done = subprocess.run( + [str(bindir / "python"), "-m", "pip", "install", "--no-index", "--no-deps", + "--disable-pip-version-check", str(self.wheel)], + capture_output=True, text=True, check=False) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + return bindir / "retrobios" + + def _same_answers(self, python: str, name: str) -> None: + command = self._installed(python, name) + for args in (["--help"], ["--version"], ["--no-such-option"]): + with self.subTest(python=name, args=args): + packaged = subprocess.run([str(command), *args], capture_output=True, + text=True, check=False, cwd=self.scratch) + script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], + capture_output=True, text=True, check=False, + cwd=self.scratch) + self.assertEqual( + (packaged.returncode, packaged.stdout, packaged.stderr), + (script.returncode, script.stdout, script.stderr)) + + def test_this_python(self): + self._same_answers(sys.executable, "venv-current") + + def test_python_38(self): + python = _python38() + if python is None: + self.skipTest("python3.8 is not on PATH") + self._same_answers(python, "venv-38") + + +def _setuptools_builds_the_sdist() -> bool: + """setuptools 77 reads the license expression the sdist declares.""" + if importlib.util.find_spec("setuptools") is None: + return False + import setuptools # noqa: PLC0415 + + return int(setuptools.__version__.split(".")[0]) >= 77 + + +class TheSdistBuildsTheSameWheel(Built): + """pip and uv build the sdist with setuptools: the files must be those the + wheel written here holds.""" + + def test_setuptools_wheel(self): + if not _setuptools_builds_the_sdist(): + if os.environ.get("CI"): + self.fail("CI installs setuptools 77 or newer") + self.skipTest("setuptools 77 or newer is not installed") + out = self.scratch / "from-sdist" + work = self.scratch / "unpacked" + with tarfile.open(self.sdist) as archive: + archive.extractall(work, filter="data") + done = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-build-isolation", "--no-deps", + "--no-index", "--disable-pip-version-check", "-w", str(out), + str(work / f"retrobios-{self.version}")], + capture_output=True, text=True, check=False) + self.assertEqual(done.returncode, 0, done.stderr[-2000:]) + (built,) = out.glob("*.whl") + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(built) as theirs, zipfile.ZipFile(self.wheel) as ours: + self.assertEqual(set(theirs.namelist()) - {f"{info}/top_level.txt"}, + set(ours.namelist())) + for name in (*MODULE_FILES, f"{info}/entry_points.txt"): + self.assertEqual(theirs.read(name), ours.read(name), name) + fields = ("Name", "Version", "Summary", "License-Expression", "Requires-Python") + theirs_meta, ours_meta = ( + email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + for archive in (theirs, ours)) + for field in fields: + self.assertEqual(theirs_meta[field], ours_meta[field], field) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/release-process.md b/wiki/release-process.md index b0367098b..71220c265 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -110,8 +110,8 @@ scripts stay Python. `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh` or `install.ps1` (the bootstraps pin the installer's SHA-256, so editing one without the other fails the run), -`ports/**` (the handheld port bundles the installer), or any input of the site -build. The path lists are spelled out once per event because the workflow +`ports/**` (the handheld port bundles the installer), `packaging/**` (the PyPI +package is built from it), or any input of the site build. The path lists are spelled out once per event because the workflow parser reads no YAML anchor, and a test holds the two equal. **Concurrency.** Per-PR group on a pull request, per-ref on a push. A pull @@ -312,6 +312,17 @@ git status --short git add release.json README.md install/*.json install/targets/*.json git commit -m "chore: record release v${DATE}" && git push +# 5b. The PyPI package, `uvx retrobios`: install.py and its command. The +# version is the date of the tag release.json now names (v2026.10.04 +# gives 2026.10.4), so a release is what bumps it. The build replaces +# any earlier one in tmp/pypi-dist. Build, run the wheel, then upload +# the two files. The token stays in its file: never in the +# repository, a script or a log. +python scripts/build_pypi.py --output-dir tmp/pypi-dist +python -m unittest tests.test_pypi_package +uvx --from tmp/pypi-dist/retrobios-*.whl retrobios --help +uv publish --token "$(cat ~/.config/retrobios/pypi-token)" tmp/pypi-dist/* + # 6. Keep only the new release plus large-files: an older pack carries hashes # the platforms no longer check, so it misleads more than it helps gh release list --json tagName,createdAt \ From 8328570d1c98529c87221ccc8c66a8f3ee33d702 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:07:54 +0100 Subject: [PATCH 2/8] docs: name uvx retrobios beside the one-liner --- README.md | 3 +++ scripts/generate_readme.py | 5 ++++- scripts/generate_site.py | 8 ++++++++ scripts/siteguides.py | 2 ++ tests/test_doc_pages.py | 2 +- wiki/installer.md | 9 +++++++++ 6 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d927af444..2001b43eb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --platform retroarch # Options in PowerShell: anything after | iex goes to iex, so the script runs as a block &([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1))) --check + +# With Python 3.8 or newer: the same installer as a package (or pipx run retrobios) +uvx retrobios ``` ![Recording of the one-line command in a terminal. The installer finds an ArkOS SD card with 6 games, plans 43 BIOS files (15.1 MB) for them, installs them and shows the summary.](docs_assets/demo/install.gif) diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index f7b8aa383..80c380910 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -39,7 +39,7 @@ from common import ( ) from nativemode import check_label import release_record -from siteguides import install_command +from siteguides import PACKAGE_COMMAND, install_command from validate_pr import LARGE_FILE_ISSUE_URL from largefiles import load_registered_paths import pr_checks @@ -633,6 +633,9 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" "", "# Options in PowerShell: anything after | iex goes to iex, so the script runs as a block", install_command("powershell", "--check"), + "", + "# With Python 3.8 or newer: the same installer as a package (or pipx run retrobios)", + PACKAGE_COMMAND, "```", "", f"![{demo_description()}]({DEMO_GIF})", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 9cda2958f..dd023a06e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -111,6 +111,7 @@ from siteguides import ( INSTALL_BASE, INSTALL_PS1, INSTALL_SH, + PACKAGE_COMMAND, TERMINALS, TRUST_ITEMS, detected_systems as _detected_systems, @@ -12013,6 +12014,13 @@ def generate_which_pack() -> str: *fence("powershell", install_command( "powershell", "--platform retroarch --dest E:\\retroarch\\cores\\system")), ]), + *markup.details("note", "Have Python? Run it as a package", [ + "The same installer, with Python 3.8 or newer:", + "", + *fence("sh", PACKAGE_COMMAND), + "", + "`pipx run retrobios` works too, or `pip install retrobios` and then `retrobios`.", + ]), *markup.details("note", "Copy the BIOS files from your old card", [ "Plug the old card in and add `--from` with its folder. The BIOS files " "found there are copied, and the rest downloaded. The old card is only read.", diff --git a/scripts/siteguides.py b/scripts/siteguides.py index a71a3edf7..12d449375 100644 --- a/scripts/siteguides.py +++ b/scripts/siteguides.py @@ -384,6 +384,8 @@ INSTALL_BASE: str = _LOADED["install_base"] # The bootstraps the site serves (sitebootstrap.py writes them). INSTALL_SH: str = f"{INSTALL_BASE}/{SH_NAME}" INSTALL_PS1: str = f"{INSTALL_BASE}/{PS1_NAME}" +# The same installer from PyPI (scripts/build_pypi.py builds the package). +PACKAGE_COMMAND = "uvx retrobios" DEVICE_GROUPS: dict[str, tuple[str, str]] = _LOADED["device_groups"] GROUP_ANCHORS: dict[str, str] = _LOADED["group_anchors"] ALSO_RUNS: dict[str, tuple[tuple[str, str, str, str, str], ...]] = _LOADED["also_runs"] diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 6ca5886f1..a17356d6e 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1566,7 +1566,7 @@ def _literals() -> str: VALUE = re.compile(r"^(?:\S*[\d/\\~<>{}=]\S*|--?[\w-]+\S*|\S*[A-Z]{2,}\S*|\.\.\.|" r"\S*\.\w{2,4}[,.:;)]?|[A-Z][a-z]+[A-Z]\S*|\(\S*)$") # A code span that is a command, a path or a shape, not a message. -NOT_A_MESSAGE = re.compile(r"^(?:\{|-|\.|/|~|<|\$|%|python|py |curl|irm|sh |pkg |sudo|ls |" +NOT_A_MESSAGE = re.compile(r"^(?:\{|-|\.|/|~|<|\$|%|python|py |curl|irm|uvx |pipx |pip |sh |pkg |sudo|ls |" r"xcode|winget|termux|install\.py|scripts/|generate_|&\()") diff --git a/wiki/installer.md b/wiki/installer.md index 894674a5d..97e8980a0 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -40,6 +40,15 @@ irm https://abdess.github.io/retrobios/i.ps1 | iex Always type the `https://`: without it the first request goes out in clear text, and anyone on the way could answer with another script. +With Python 3.8 or newer, the same installer is a package: + +```bash +uvx retrobios +``` + +`pipx run retrobios` works too, or `pip install retrobios` and then +`retrobios`. Options follow the command: `uvx retrobios --check`. + `i` and `i.ps1` are `install.sh` and `install.ps1` as of the commit the site was built from, set to download `install.py` from that same commit. The fingerprint they check and the installer they fetch never come from two From 0c2bfd75f6e49c6f4478b3e4062252c5d9cd07de Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:35:52 +0100 Subject: [PATCH 3/8] docs: list the PyPI test, drop pronouns there --- packaging/pypi/README.md | 4 ++-- packaging/pypi/pyproject.toml | 2 +- wiki/testing-guide.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/pypi/README.md b/packaging/pypi/README.md index 98f85de7a..f2c1a0f9e 100644 --- a/packaging/pypi/README.md +++ b/packaging/pypi/README.md @@ -1,8 +1,8 @@ # RetroBIOS -Puts the BIOS files your emulators need in the right folder. +Puts the BIOS files emulators need in the right folder, and checks them. -It finds your setup and its BIOS folder, downloads only the missing files and +It finds the setup and its BIOS folder, downloads only the missing files and checks each one. It works with RetroArch, Batocera, EmuDeck, RetroBat, Recalbox, RetroDECK, RomM and more. diff --git a/packaging/pypi/pyproject.toml b/packaging/pypi/pyproject.toml index eeb1d92f4..045079edf 100644 --- a/packaging/pypi/pyproject.toml +++ b/packaging/pypi/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "retrobios" dynamic = ["version"] -description = "Install the BIOS files your emulators need" +description = "Install the BIOS files emulators need, checked" readme = "README.md" requires-python = ">=3.8" license = "MIT" diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 23197370a..c2329ead9 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -59,7 +59,7 @@ Every module in `tests/`, by the area it guards. | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_emuelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | -| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` | +| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui`, `test_pypi_package` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` | | Measurement records | synthetic record files, the committed `measurements/` against HEAD, the **Proof** line naming the BIOS a core ran, and that no page shows a Tested block, a region outcome or a time per frame | `test_measurements` | From 4053aa2601fa15131d4f2f0e7ba721d70b1da9de Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:28:04 +0100 Subject: [PATCH 4/8] docs: put uvx retrobios in the macOS, Linux tabs --- scripts/generate_site.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index e2cf34562..c213c63c4 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -11951,6 +11951,11 @@ def generate_which_pack() -> str: # handheld recording (sitesteps). box_tabs = (*all_tabs, "device") box_steps, box_demos = sitesteps.install_box(box_tabs) + # Python is on most Linux and macOS machines: the package runs the same + # installer there without the bootstrap. + for tab in ("macos", "linux"): + box_demos[tab] = [*box_demos[tab], + f"With Python 3.8 or newer, `{PACKAGE_COMMAND}` does the same.", ""] lines = [ "---", f"terms: {', '.join(IOS_TERMS)}", @@ -12001,8 +12006,6 @@ def generate_which_pack() -> str: "On Batocera, Recalbox and other game systems, run it on the device. " "The setup's page shows how.", "", - f"With Python, `{PACKAGE_COMMAND}` does the same.", - "", *first_screen_words(), "", *trust_box(""), From 49873497ba4000ed1ae3b1cf686eeba9bac8e018 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:30:22 +0100 Subject: [PATCH 5/8] fix: deploy the site when sitesteps.py changes --- .github/workflows/deploy-site.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 0f4721b23..87953b4ab 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -60,6 +60,7 @@ on: - "scripts/siterender.py" - "scripts/retired_files.py" - "scripts/sitesources.py" + - "scripts/sitesteps.py" - "scripts/sitestarlight.py" - "scripts/sitewrite.py" - "scripts/slot.py" From 7da48ca56a6dba3cd4861c3adb4d43f9817891ca Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:37 +0100 Subject: [PATCH 6/8] feat: date the PyPI package by install.py commits --- scripts/build_pypi.py | 37 +++++++++++++++++++++++++------------ tests/test_pypi_package.py | 20 +++++++++++++------- wiki/release-process.md | 12 +++++++----- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/scripts/build_pypi.py b/scripts/build_pypi.py index efbeca92a..d7b3e7167 100644 --- a/scripts/build_pypi.py +++ b/scripts/build_pypi.py @@ -6,9 +6,11 @@ bootstraps pin, a two-line launcher that runs it as a script, the metadata, README.md and LICENSE. Nothing of the collection. The metadata comes from packaging/pypi/pyproject.toml. -The version is the date of the release in release.json: v2026.10.10 gives -2026.10.10. A release writes release.json first, so the package published -with it carries its date. +The version is the UTC date of the last commit that changed install.py, +with the zeros PEP 440 drops left out: 2026-11-09 gives 2026.11.9. When +install.py changed several times that day, the newest gets .N, its rank in +the day: the third change gives 2026.11.9.3. The same install.py always +gives the same version, so PyPI refuses a republish of unchanged code. Both archives are written here with the standard library rather than by a build backend: the bytes then depend on the inputs alone, on any machine and @@ -25,11 +27,13 @@ from __future__ import annotations import argparse import base64 import calendar +import datetime import gzip import hashlib import io import json import re +import subprocess import tarfile import zipfile from pathlib import Path @@ -63,13 +67,22 @@ PROJECT_KEYS = frozenset({ }) -def package_version(release: dict) -> str: - """The release date as a version, without the zeros PEP 440 drops.""" - tag = str(release.get("tag", "")) - match = re.fullmatch(r"v(\d{4})\.(\d{2})\.(\d{2})", tag) - if match is None: - raise ValueError(f"release.json names no dated release tag: {tag!r}") - return ".".join(str(int(part)) for part in match.groups()) +def package_version(stamps: list[int]) -> str: + """The version of install.py from its commit times, newest first (UTC).""" + if not stamps: + raise ValueError("install.py has no commit to date the package by") + day = datetime.datetime.fromtimestamp(stamps[0], datetime.timezone.utc).date() + rank = sum(1 for stamp in stamps + if datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).date() == day) + base = f"{day.year}.{day.month}.{day.day}" + return base if rank == 1 else f"{base}.{rank}" + + +def install_py_commits() -> list[int]: + """Commit times of install.py, newest first, from git.""" + out = subprocess.run(["git", "log", "--format=%ct", "--", "install.py"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout + return [int(line) for line in out.split()] def load_project() -> dict: @@ -181,8 +194,8 @@ def write_sdist(path: Path, project: dict, files: dict[str, bytes], version: str def build(output_dir: Path) -> tuple[Path, Path]: """The wheel and the sdist in output_dir, replacing earlier builds there.""" - version = package_version(json.loads((ROOT / "release.json").read_text(encoding="utf-8"))) - year, month, day = (int(part) for part in version.split(".")) + version = package_version(install_py_commits()) + year, month, day = (int(part) for part in version.split(".")[:3]) stamp = (year, month, day) project, files = package_files(version) output_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_pypi_package.py b/tests/test_pypi_package.py index cf3e7bdb5..812468b3e 100644 --- a/tests/test_pypi_package.py +++ b/tests/test_pypi_package.py @@ -96,15 +96,21 @@ class TheArchivesHoldTheInstallerAndNothingElse(Built): [self.wheel.read_bytes(), self.sdist.read_bytes()]) -class TheVersionIsTheReleaseDate(unittest.TestCase): +class TheVersionIsInstallPyCommitDate(unittest.TestCase): + """The release date put November code under an October version: the + version now follows the commits of install.py itself.""" - def test_dated_tag(self): - self.assertEqual(build_pypi.package_version({"tag": "v2026.10.04"}), "2026.10.4") + def test_one_change_that_day(self): + # 2026-11-09 12:00 UTC, then a change the day before. + self.assertEqual(build_pypi.package_version([1794225600, 1794139200]), "2026.11.9") - def test_other_tag_refused(self): - for tag in ("large-files", "2026.10.04", "v2026.10"): - with self.assertRaises(ValueError): - build_pypi.package_version({"tag": tag}) + def test_the_rank_in_the_day(self): + self.assertEqual(build_pypi.package_version([1794229200, 1794225600, 1794222000]), + "2026.11.9.3") + + def test_no_commit_refused(self): + with self.assertRaises(ValueError): + build_pypi.package_version([]) def _python38() -> str | None: diff --git a/wiki/release-process.md b/wiki/release-process.md index bafe163e7..0914e4a71 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -313,11 +313,13 @@ git add release.json README.md install/*.json install/targets/*.json git commit -m "chore: record release v${DATE}" && git push # 5b. The PyPI package, `uvx retrobios`: install.py and its command. The -# version is the date of the tag release.json now names (v2026.10.04 -# gives 2026.10.4), so a release is what bumps it. The build replaces -# any earlier one in tmp/pypi-dist. Build, run the wheel, then upload -# the two files. The token stays in its file: never in the -# repository, a script or a log. +# version is the UTC date of the last commit that changed install.py +# (2026-11-09 gives 2026.11.9, the third change that day 2026.11.9.3), +# so an unchanged install.py keeps its version and PyPI refuses it +# again. Publishing is not tied to a release: any push that changes +# install.py can be published. The build replaces any earlier one in +# tmp/pypi-dist. Build, run the wheel, then upload the two files. The +# token stays in its file: never in the repository, a script or a log. python scripts/build_pypi.py --output-dir tmp/pypi-dist python -m unittest tests.test_pypi_package uvx --from tmp/pypi-dist/retrobios-*.whl retrobios --help From 277ffb707970fae07b64838da9f7a83250049062 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:40:37 +0100 Subject: [PATCH 7/8] chore: name the contributor project retrobios-dev --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bf5a325ab..d81cd52c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,10 @@ packages = [] py-modules = [] [project] -name = "retrobios" +# Not the package PyPI serves (packaging/pypi/ builds that one, named +# retrobios): a contributor's `pip install -e ".[dev]"` and `pip install +# retrobios` in one environment would otherwise replace each other. +name = "retrobios-dev" version = "1.0.0" description = "BIOS collection manager for retrogaming platforms" # install.py is the only entry point a user runs before cloning, and both From b9a35589a8e6f0ee785c755e47e68423624f8209 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:45:49 +0100 Subject: [PATCH 8/8] feat: build the PyPI package with flit_core build_pypi.py stages install.py, the launcher, __main__.py, README and LICENSE, then runs uv build with flit_core 3.12.0 pinned and SOURCE_DATE_EPOCH set from install.py's commit. Adds python -m retrobios, the PyPA classifiers, keywords and project URLs. CI installs uv. --- .github/workflows/validate.yml | 5 +- CONTRIBUTING.md | 2 +- packaging/pypi/pyproject.toml | 36 ++++-- packaging/pypi/retrobios/__main__.py | 5 + pyproject.toml | 2 + scripts/build_pypi.py | 166 +++++++-------------------- tests/test_pypi_package.py | 130 +++++++++++---------- wiki/release-process.md | 4 +- 8 files changed, 147 insertions(+), 203 deletions(-) create mode 100644 packaging/pypi/retrobios/__main__.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 06853c125..80d91a418 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -244,9 +244,10 @@ jobs: # heading guard's premise, and skipped on every run without it. pyte # reads the installer's full screen back in its end-to-end tests, # hypothesis fuzzes its key decoder, and vermin holds install.py to 3.8. - # setuptools reads pyproject.toml for the packaging guard. + # setuptools reads pyproject.toml for the packaging guard, and uv builds + # the PyPI package the way a release builds it. - name: Install dependencies - run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" + run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" "uv==0.12.9" # The full screen also runs under the 3.8 floor: the end-to-end tests # start install.py with the interpreter that runs them. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7acb100ce..f919b82c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,7 @@ validation above is its check. For code, profiles or platform files, these are the checks CI runs on a pull request (Python 3.12 or newer): ```bash -pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" +pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" "uv==0.12.9" python scripts/pr_checks.py ``` diff --git a/packaging/pypi/pyproject.toml b/packaging/pypi/pyproject.toml index 045079edf..fe52886c3 100644 --- a/packaging/pypi/pyproject.toml +++ b/packaging/pypi/pyproject.toml @@ -1,11 +1,14 @@ # The package PyPI serves as `retrobios`: install.py and the command that # runs it, nothing else. The root pyproject.toml is the contributors' -# environment. scripts/build_pypi.py writes the wheel and the sdist from this -# file, README.md, LICENSE and install.py, and writes retrobios/__init__.py -# with the version. This file is the one a build from the sdist reads. +# environment. scripts/build_pypi.py stages this file, README.md, LICENSE, +# retrobios/__main__.py and install.py into a build folder, writes +# retrobios/__init__.py with the version, and runs `uv build` there. [build-system] -requires = ["setuptools>=77"] -build-backend = "setuptools.build_meta" +# Pinned: the same backend writes the same bytes from the same files. 3.12 +# writes core metadata 2.4, the version this package was checked under +# (twine check --strict). 4.x writes 2.5. +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" [project] name = "retrobios" @@ -16,22 +19,33 @@ requires-python = ">=3.8" license = "MIT" license-files = ["LICENSE"] dependencies = [] +keywords = [ + "bios", "firmware", "emulation", "emulator", "retrogaming", "installer", + "retroarch", "batocera", "emudeck", "retrobat", "recalbox", "retrodeck", +] classifiers = [ + "Development Status :: 5 - Production/Stable", "Environment :: Console", + "Intended Audience :: End Users/Desktop", "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: System :: Emulators", ] [project.urls] Homepage = "https://abdess.github.io/retrobios/" +Documentation = "https://abdess.github.io/retrobios/wiki/installer/" Source = "https://github.com/Abdess/retrobios" +Issues = "https://github.com/Abdess/retrobios/issues" +Releases = "https://github.com/Abdess/retrobios/releases" [project.scripts] retrobios = "retrobios:main" - -[tool.setuptools] -packages = ["retrobios"] - -[tool.setuptools.dynamic] -version = { attr = "retrobios.__version__" } diff --git a/packaging/pypi/retrobios/__main__.py b/packaging/pypi/retrobios/__main__.py new file mode 100644 index 000000000..e73f79f0a --- /dev/null +++ b/packaging/pypi/retrobios/__main__.py @@ -0,0 +1,5 @@ +"""`python -m retrobios` runs the installer as the retrobios command does.""" + +from retrobios import main + +main() diff --git a/pyproject.toml b/pyproject.toml index d81cd52c0..ca2e08700 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,8 @@ dev = [ # installs only in its isolated build environment and 3.12 no longer # bundles: without it the guard skipped on every CI run. "setuptools>=61", + # scripts/build_pypi.py runs `uv build` for the PyPI package and its test. + "uv==0.12.9", # The type check CI runs, pinned: another mypy reports other errors # than the ones mypy-baseline.txt holds. "mypy==1.18.2", diff --git a/scripts/build_pypi.py b/scripts/build_pypi.py index d7b3e7167..93dbf262f 100644 --- a/scripts/build_pypi.py +++ b/scripts/build_pypi.py @@ -2,20 +2,20 @@ """Build the `retrobios` package PyPI serves: install.py as a command. The wheel and the sdist hold install.py, byte for byte the copy the -bootstraps pin, a two-line launcher that runs it as a script, the metadata, -README.md and LICENSE. Nothing of the collection. The metadata comes from -packaging/pypi/pyproject.toml. +bootstraps pin, a launcher that runs it as a script (`retrobios` and +`python -m retrobios`), the metadata, README.md and LICENSE. Nothing of the +collection. packaging/pypi/ holds the project file, the README and +`__main__.py`; this script stages them with install.py and LICENSE into a +build folder, writes `__init__.py` with the version, and runs `uv build` +there with the flit_core backend pinned in the project file. The version is the UTC date of the last commit that changed install.py, with the zeros PEP 440 drops left out: 2026-11-09 gives 2026.11.9. When install.py changed several times that day, the newest gets .N, its rank in the day: the third change gives 2026.11.9.3. The same install.py always gives the same version, so PyPI refuses a republish of unchanged code. - -Both archives are written here with the standard library rather than by a -build backend: the bytes then depend on the inputs alone, on any machine and -with no network. The sdist still names setuptools as its backend, for pip -and uv to build it. +SOURCE_DATE_EPOCH is that commit's time, so two builds of one commit give +the same bytes. Usage: python scripts/build_pypi.py @@ -25,17 +25,12 @@ Usage: from __future__ import annotations import argparse -import base64 -import calendar import datetime -import gzip import hashlib -import io -import json -import re +import os +import shutil import subprocess -import tarfile -import zipfile +import tempfile from pathlib import Path from sitebootstrap import PIN, SH_NAME @@ -59,13 +54,6 @@ def main() -> None: runpy.run_path(str(Path(__file__).with_name("install.py")), run_name="__main__") ''' -# The [project] keys the metadata below writes. Another key in pyproject.toml -# would reach a build from the sdist and not the wheel written here. -PROJECT_KEYS = frozenset({ - "name", "dynamic", "description", "readme", "requires-python", "license", - "license-files", "dependencies", "classifiers", "urls", "scripts", -}) - def package_version(stamps: list[int]) -> str: """The version of install.py from its commit times, newest first (UTC).""" @@ -85,36 +73,6 @@ def install_py_commits() -> list[int]: return [int(line) for line in out.split()] -def load_project() -> dict: - import tomllib # noqa: PLC0415 # 3.11, only where a package is built - - with (PACKAGE_DIR / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - unknown = set(project) - PROJECT_KEYS - if unknown: - raise ValueError(f"build_pypi.py writes no metadata for {sorted(unknown)}") - if project["dependencies"] or project["dynamic"] != ["version"]: - raise ValueError("the package has no dependencies and only its version is dynamic") - return project - - -def metadata(project: dict, version: str, readme: str) -> str: - """Core metadata 2.4, as the wheel's METADATA and the sdist's PKG-INFO.""" - lines = [ - "Metadata-Version: 2.4", - f"Name: {project['name']}", - f"Version: {version}", - f"Summary: {project['description']}", - *(f"Project-URL: {label}, {url}" for label, url in project["urls"].items()), - f"License-Expression: {project['license']}", - *(f"License-File: {name}" for name in project["license-files"]), - *(f"Classifier: {name}" for name in project["classifiers"]), - f"Requires-Python: {project['requires-python']}", - "Description-Content-Type: text/markdown", - ] - return "\n".join(lines) + "\n\n" + readme - - def installer_bytes() -> bytes: """install.py, refused unless it is the one install.sh pins: the package then runs the installer the one-line command runs, under the same id.""" @@ -125,87 +83,41 @@ def installer_bytes() -> bytes: return data -def package_files(version: str) -> tuple[dict, dict[str, bytes]]: - """The project table, and the files both archives ship, by path.""" - project = load_project() - readme = (PACKAGE_DIR / project["readme"]).read_text(encoding="utf-8") - files = { - f"{MODULE}/__init__.py": LAUNCHER.format(version=version).encode(), - f"{MODULE}/install.py": installer_bytes(), - "METADATA": metadata(project, version, readme).encode("utf-8"), - "README.md": readme.encode("utf-8"), - "pyproject.toml": (PACKAGE_DIR / "pyproject.toml").read_bytes(), - } - for name in project["license-files"]: - files[name] = (ROOT / name).read_bytes() - return project, files - - -def _record_line(path: str, data: bytes) -> str: - digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() - return f"{path},sha256={digest},{len(data)}" - - -def write_wheel(path: Path, project: dict, files: dict[str, bytes], version: str, - stamp: tuple[int, int, int]) -> None: - info = f"{project['name']}-{version}.dist-info" - scripts = "".join(f"{name} = {target}\n" for name, target in project["scripts"].items()) - members = { - f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], - f"{MODULE}/install.py": files[f"{MODULE}/install.py"], - f"{info}/METADATA": files["METADATA"], - f"{info}/WHEEL": (b"Wheel-Version: 1.0\nGenerator: retrobios build_pypi.py\n" - b"Root-Is-Purelib: true\nTag: py3-none-any\n"), - f"{info}/entry_points.txt": f"[console_scripts]\n{scripts}".encode(), - **{f"{info}/licenses/{name}": files[name] for name in project["license-files"]}, - } - record = "".join(_record_line(name, data) + "\n" for name, data in members.items()) - members[f"{info}/RECORD"] = (record + f"{info}/RECORD,,\n").encode() - with zipfile.ZipFile(path, "w") as archive: - for name, data in members.items(): - entry = zipfile.ZipInfo(name, date_time=(*stamp, 0, 0, 0)) - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = 0o644 << 16 - archive.writestr(entry, data) - - -def write_sdist(path: Path, project: dict, files: dict[str, bytes], version: str, - stamp: tuple[int, int, int]) -> None: - top = f"{project['name']}-{version}" - members = { - "PKG-INFO": files["METADATA"], - "pyproject.toml": files["pyproject.toml"], - "README.md": files["README.md"], - **{name: files[name] for name in project["license-files"]}, - f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], - f"{MODULE}/install.py": files[f"{MODULE}/install.py"], - } - mtime = calendar.timegm((*stamp, 0, 0, 0)) - raw = io.BytesIO() - with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as archive: - for name, data in members.items(): - entry = tarfile.TarInfo(f"{top}/{name}") - entry.size, entry.mtime, entry.mode = len(data), mtime, 0o644 - archive.addfile(entry, io.BytesIO(data)) - with path.open("wb") as handle, gzip.GzipFile(fileobj=handle, mode="wb", filename="", - mtime=mtime) as packed: - packed.write(raw.getvalue()) +def stage(folder: Path, version: str) -> None: + """The project as the backend reads it.""" + module = folder / MODULE + module.mkdir(parents=True) + for name in ("pyproject.toml", "README.md"): + shutil.copyfile(PACKAGE_DIR / name, folder / name) + shutil.copyfile(ROOT / "LICENSE", folder / "LICENSE") + shutil.copyfile(PACKAGE_DIR / MODULE / "__main__.py", module / "__main__.py") + (module / "__init__.py").write_text(LAUNCHER.format(version=version), encoding="utf-8") + (module / "install.py").write_bytes(installer_bytes()) def build(output_dir: Path) -> tuple[Path, Path]: """The wheel and the sdist in output_dir, replacing earlier builds there.""" - version = package_version(install_py_commits()) - year, month, day = (int(part) for part in version.split(".")[:3]) - stamp = (year, month, day) - project, files = package_files(version) + stamps = install_py_commits() + version = package_version(stamps) output_dir.mkdir(parents=True, exist_ok=True) for pattern in ("*.whl", "*.tar.gz"): - for old in output_dir.glob(f"{project['name']}-{pattern}"): + for old in output_dir.glob(f"{MODULE}-{pattern}"): old.unlink() - wheel = output_dir / f"{project['name']}-{version}-py3-none-any.whl" - sdist = output_dir / f"{project['name']}-{version}.tar.gz" - write_wheel(wheel, project, files, version, stamp) - write_sdist(sdist, project, files, version, stamp) + scratch = ROOT / "tmp" + scratch.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pypi-stage-", dir=scratch) as name: + folder = Path(name) / "retrobios" + stage(folder, version) + # Without a flag, uv builds the sdist, then the wheel from that sdist: + # what a user building from the sdist gets is what is published. + subprocess.run( + ["uv", "build", "--quiet", "--out-dir", str(output_dir.resolve()), str(folder)], + check=True, env={**os.environ, "SOURCE_DATE_EPOCH": str(stamps[0])}) + wheel = output_dir / f"{MODULE}-{version}-py3-none-any.whl" + sdist = output_dir / f"{MODULE}-{version}.tar.gz" + for path in (wheel, sdist): + if not path.is_file(): + raise RuntimeError(f"uv build wrote no {path.name}") return wheel, sdist diff --git a/tests/test_pypi_package.py b/tests/test_pypi_package.py index 812468b3e..4e49f5f69 100644 --- a/tests/test_pypi_package.py +++ b/tests/test_pypi_package.py @@ -2,16 +2,17 @@ `uvx retrobios`, `pipx run retrobios` and `pip install retrobios` reach the installer the one-line command runs: the same bytes, so the same id, the -same file lists, cache and exit codes. scripts/build_pypi.py writes the -wheel and the sdist, and these tests read what it wrote and run it. +same file lists, cache and exit codes. scripts/build_pypi.py stages the +project and builds it with `uv build` and the flit_core backend, and these +tests read what it wrote and run it. """ from __future__ import annotations import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache +import configparser import email.parser -import importlib.util import os import shutil import subprocess @@ -19,7 +20,6 @@ import sys import tarfile import tempfile import unittest -import venv import zipfile from pathlib import Path @@ -28,7 +28,7 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import build_pypi # noqa: E402 -MODULE_FILES = {"retrobios/__init__.py", "retrobios/install.py"} +MODULE_FILES = {"retrobios/__init__.py", "retrobios/__main__.py", "retrobios/install.py"} class Built(unittest.TestCase): @@ -42,6 +42,10 @@ class Built(unittest.TestCase): @classmethod def setUpClass(cls) -> None: + if shutil.which("uv") is None: + if os.environ.get("CI"): + raise AssertionError("CI installs uv") + raise unittest.SkipTest("uv is not installed") cls._scratch = tempfile.TemporaryDirectory(prefix="pypi-") cls.scratch = Path(cls._scratch.name) cls.wheel, cls.sdist = build_pypi.build(cls.scratch / "dist") @@ -65,13 +69,38 @@ class TheArchivesHoldTheInstallerAndNothingElse(Built): (REPO_ROOT / "install.py").read_bytes()) entry_points = archive.read(f"{info}/entry_points.txt").decode() meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) - self.assertEqual(entry_points, "[console_scripts]\nretrobios = retrobios:main\n") + scripts = configparser.ConfigParser() + scripts.read_string(entry_points) + self.assertEqual(dict(scripts["console_scripts"]), {"retrobios": "retrobios:main"}) self.assertEqual(meta["Name"], "retrobios") self.assertEqual(meta["Version"], self.version) self.assertEqual(meta["Requires-Python"], ">=3.8") self.assertIsNone(meta["Requires-Dist"]) - self.assertEqual(meta.get_payload(), - (REPO_ROOT / "packaging/pypi/README.md").read_text(encoding="utf-8")) + # The backend ends the body with one more newline than the file. + self.assertEqual(meta.get_payload().rstrip("\n"), + (REPO_ROOT / "packaging/pypi/README.md").read_text( + encoding="utf-8").rstrip("\n")) + + def test_metadata(self): + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(self.wheel) as archive: + meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + self.assertEqual(meta["Metadata-Version"], "2.4") + self.assertEqual(meta["License-Expression"], "MIT") + classifiers = meta.get_all("Classifier") + # PEP 639: the license is the expression, never a classifier as well. + self.assertEqual([name for name in classifiers if name.startswith("License ::")], []) + self.assertEqual( + [name for name in classifiers if name.startswith("Programming Language :: Python :: 3.")], + [f"Programming Language :: Python :: 3.{minor}" for minor in range(8, 15)]) + for name in ("Programming Language :: Python :: 3 :: Only", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: End Users/Desktop", "Environment :: Console", + "Operating System :: OS Independent", "Topic :: System :: Emulators"): + self.assertIn(name, classifiers) + self.assertTrue(meta["Keywords"]) + self.assertEqual(sorted(url.split(",")[0] for url in meta.get_all("Project-URL")), + ["Documentation", "Homepage", "Issues", "Releases", "Source"]) def test_sdist_members(self): top = f"retrobios-{self.version}" @@ -91,6 +120,8 @@ class TheArchivesHoldTheInstallerAndNothingElse(Built): self.assertEqual([name for name in names if name.startswith(top)], [], top) def test_build_is_reproducible(self): + # SOURCE_DATE_EPOCH is install.py's commit time, so the clock moving + # between the two builds changes no byte. again = build_pypi.build(self.scratch / "again") self.assertEqual([path.read_bytes() for path in again], [self.wheel.read_bytes(), self.sdist.read_bytes()]) @@ -113,6 +144,23 @@ class TheVersionIsInstallPyCommitDate(unittest.TestCase): build_pypi.package_version([]) +class AnUnpinnedInstallerIsRefused(unittest.TestCase): + """The package runs the installer the one-line command runs, or none.""" + + def test_pin_mismatch(self): + with tempfile.TemporaryDirectory(prefix="pypi-pin-") as name: + root = Path(name) + (root / "install.py").write_bytes(b"print('another')\n") + shutil.copyfile(REPO_ROOT / "install.sh", root / "install.sh") + original = build_pypi.ROOT + build_pypi.ROOT = root + try: + with self.assertRaises(ValueError): + build_pypi.installer_bytes() + finally: + build_pypi.ROOT = original + + def _python38() -> str | None: found = shutil.which("python3.8") if found is None and os.environ.get("CI"): @@ -138,16 +186,19 @@ class TheInstalledCommandIsTheInstaller(Built): def _same_answers(self, python: str, name: str) -> None: command = self._installed(python, name) - for args in (["--help"], ["--version"], ["--no-such-option"]): - with self.subTest(python=name, args=args): - packaged = subprocess.run([str(command), *args], capture_output=True, - text=True, check=False, cwd=self.scratch) - script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], - capture_output=True, text=True, check=False, - cwd=self.scratch) - self.assertEqual( - (packaged.returncode, packaged.stdout, packaged.stderr), - (script.returncode, script.stdout, script.stderr)) + # `retrobios` and `python -m retrobios` both run install.py. + launchers = ([str(command)], [str(command.with_name("python")), "-m", "retrobios"]) + for launcher in launchers: + for args in (["--help"], ["--version"], ["--no-such-option"]): + with self.subTest(python=name, launcher=launcher[-1], args=args): + packaged = subprocess.run([*launcher, *args], capture_output=True, + text=True, check=False, cwd=self.scratch) + script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], + capture_output=True, text=True, check=False, + cwd=self.scratch) + self.assertEqual( + (packaged.returncode, packaged.stdout, packaged.stderr), + (script.returncode, script.stdout, script.stderr)) def test_this_python(self): self._same_answers(sys.executable, "venv-current") @@ -159,48 +210,5 @@ class TheInstalledCommandIsTheInstaller(Built): self._same_answers(python, "venv-38") -def _setuptools_builds_the_sdist() -> bool: - """setuptools 77 reads the license expression the sdist declares.""" - if importlib.util.find_spec("setuptools") is None: - return False - import setuptools # noqa: PLC0415 - - return int(setuptools.__version__.split(".")[0]) >= 77 - - -class TheSdistBuildsTheSameWheel(Built): - """pip and uv build the sdist with setuptools: the files must be those the - wheel written here holds.""" - - def test_setuptools_wheel(self): - if not _setuptools_builds_the_sdist(): - if os.environ.get("CI"): - self.fail("CI installs setuptools 77 or newer") - self.skipTest("setuptools 77 or newer is not installed") - out = self.scratch / "from-sdist" - work = self.scratch / "unpacked" - with tarfile.open(self.sdist) as archive: - archive.extractall(work, filter="data") - done = subprocess.run( - [sys.executable, "-m", "pip", "wheel", "--no-build-isolation", "--no-deps", - "--no-index", "--disable-pip-version-check", "-w", str(out), - str(work / f"retrobios-{self.version}")], - capture_output=True, text=True, check=False) - self.assertEqual(done.returncode, 0, done.stderr[-2000:]) - (built,) = out.glob("*.whl") - info = f"retrobios-{self.version}.dist-info" - with zipfile.ZipFile(built) as theirs, zipfile.ZipFile(self.wheel) as ours: - self.assertEqual(set(theirs.namelist()) - {f"{info}/top_level.txt"}, - set(ours.namelist())) - for name in (*MODULE_FILES, f"{info}/entry_points.txt"): - self.assertEqual(theirs.read(name), ours.read(name), name) - fields = ("Name", "Version", "Summary", "License-Expression", "Requires-Python") - theirs_meta, ours_meta = ( - email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) - for archive in (theirs, ours)) - for field in fields: - self.assertEqual(theirs_meta[field], ours_meta[field], field) - - if __name__ == "__main__": unittest.main() diff --git a/wiki/release-process.md b/wiki/release-process.md index 0914e4a71..6b935e380 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -318,7 +318,9 @@ git commit -m "chore: record release v${DATE}" && git push # so an unchanged install.py keeps its version and PyPI refuses it # again. Publishing is not tied to a release: any push that changes # install.py can be published. The build replaces any earlier one in -# tmp/pypi-dist. Build, run the wheel, then upload the two files. The +# tmp/pypi-dist. `uv build` builds it with the flit_core backend the +# project pins, dated by that commit (SOURCE_DATE_EPOCH), so a rebuild +# gives the same bytes. Build, run the wheel, then upload the two files. The # token stays in its file: never in the repository, a script or a log. python scripts/build_pypi.py --output-dir tmp/pypi-dist python -m unittest tests.test_pypi_package