diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 06853c125..80d91a418 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -244,9 +244,10 @@ jobs: # heading guard's premise, and skipped on every run without it. pyte # reads the installer's full screen back in its end-to-end tests, # hypothesis fuzzes its key decoder, and vermin holds install.py to 3.8. - # setuptools reads pyproject.toml for the packaging guard. + # setuptools reads pyproject.toml for the packaging guard, and uv builds + # the PyPI package the way a release builds it. - name: Install dependencies - run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" + run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" "uv==0.12.9" # The full screen also runs under the 3.8 floor: the end-to-end tests # start install.py with the interpreter that runs them. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7acb100ce..f919b82c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,7 @@ validation above is its check. For code, profiles or platform files, these are the checks CI runs on a pull request (Python 3.12 or newer): ```bash -pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" +pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "pyte==0.8.2" "hypothesis==6.170.8" "vermin==1.9.2" "setuptools==80.9.0" "uv==0.12.9" python scripts/pr_checks.py ``` diff --git a/packaging/pypi/pyproject.toml b/packaging/pypi/pyproject.toml index 045079edf..fe52886c3 100644 --- a/packaging/pypi/pyproject.toml +++ b/packaging/pypi/pyproject.toml @@ -1,11 +1,14 @@ # The package PyPI serves as `retrobios`: install.py and the command that # runs it, nothing else. The root pyproject.toml is the contributors' -# environment. scripts/build_pypi.py writes the wheel and the sdist from this -# file, README.md, LICENSE and install.py, and writes retrobios/__init__.py -# with the version. This file is the one a build from the sdist reads. +# environment. scripts/build_pypi.py stages this file, README.md, LICENSE, +# retrobios/__main__.py and install.py into a build folder, writes +# retrobios/__init__.py with the version, and runs `uv build` there. [build-system] -requires = ["setuptools>=77"] -build-backend = "setuptools.build_meta" +# Pinned: the same backend writes the same bytes from the same files. 3.12 +# writes core metadata 2.4, the version this package was checked under +# (twine check --strict). 4.x writes 2.5. +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" [project] name = "retrobios" @@ -16,22 +19,33 @@ requires-python = ">=3.8" license = "MIT" license-files = ["LICENSE"] dependencies = [] +keywords = [ + "bios", "firmware", "emulation", "emulator", "retrogaming", "installer", + "retroarch", "batocera", "emudeck", "retrobat", "recalbox", "retrodeck", +] classifiers = [ + "Development Status :: 5 - Production/Stable", "Environment :: Console", + "Intended Audience :: End Users/Desktop", "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: System :: Emulators", ] [project.urls] Homepage = "https://abdess.github.io/retrobios/" +Documentation = "https://abdess.github.io/retrobios/wiki/installer/" Source = "https://github.com/Abdess/retrobios" +Issues = "https://github.com/Abdess/retrobios/issues" +Releases = "https://github.com/Abdess/retrobios/releases" [project.scripts] retrobios = "retrobios:main" - -[tool.setuptools] -packages = ["retrobios"] - -[tool.setuptools.dynamic] -version = { attr = "retrobios.__version__" } diff --git a/packaging/pypi/retrobios/__main__.py b/packaging/pypi/retrobios/__main__.py new file mode 100644 index 000000000..e73f79f0a --- /dev/null +++ b/packaging/pypi/retrobios/__main__.py @@ -0,0 +1,5 @@ +"""`python -m retrobios` runs the installer as the retrobios command does.""" + +from retrobios import main + +main() diff --git a/pyproject.toml b/pyproject.toml index d81cd52c0..ca2e08700 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,8 @@ dev = [ # installs only in its isolated build environment and 3.12 no longer # bundles: without it the guard skipped on every CI run. "setuptools>=61", + # scripts/build_pypi.py runs `uv build` for the PyPI package and its test. + "uv==0.12.9", # The type check CI runs, pinned: another mypy reports other errors # than the ones mypy-baseline.txt holds. "mypy==1.18.2", diff --git a/scripts/build_pypi.py b/scripts/build_pypi.py index d7b3e7167..93dbf262f 100644 --- a/scripts/build_pypi.py +++ b/scripts/build_pypi.py @@ -2,20 +2,20 @@ """Build the `retrobios` package PyPI serves: install.py as a command. The wheel and the sdist hold install.py, byte for byte the copy the -bootstraps pin, a two-line launcher that runs it as a script, the metadata, -README.md and LICENSE. Nothing of the collection. The metadata comes from -packaging/pypi/pyproject.toml. +bootstraps pin, a launcher that runs it as a script (`retrobios` and +`python -m retrobios`), the metadata, README.md and LICENSE. Nothing of the +collection. packaging/pypi/ holds the project file, the README and +`__main__.py`; this script stages them with install.py and LICENSE into a +build folder, writes `__init__.py` with the version, and runs `uv build` +there with the flit_core backend pinned in the project file. The version is the UTC date of the last commit that changed install.py, with the zeros PEP 440 drops left out: 2026-11-09 gives 2026.11.9. When install.py changed several times that day, the newest gets .N, its rank in the day: the third change gives 2026.11.9.3. The same install.py always gives the same version, so PyPI refuses a republish of unchanged code. - -Both archives are written here with the standard library rather than by a -build backend: the bytes then depend on the inputs alone, on any machine and -with no network. The sdist still names setuptools as its backend, for pip -and uv to build it. +SOURCE_DATE_EPOCH is that commit's time, so two builds of one commit give +the same bytes. Usage: python scripts/build_pypi.py @@ -25,17 +25,12 @@ Usage: from __future__ import annotations import argparse -import base64 -import calendar import datetime -import gzip import hashlib -import io -import json -import re +import os +import shutil import subprocess -import tarfile -import zipfile +import tempfile from pathlib import Path from sitebootstrap import PIN, SH_NAME @@ -59,13 +54,6 @@ def main() -> None: runpy.run_path(str(Path(__file__).with_name("install.py")), run_name="__main__") ''' -# The [project] keys the metadata below writes. Another key in pyproject.toml -# would reach a build from the sdist and not the wheel written here. -PROJECT_KEYS = frozenset({ - "name", "dynamic", "description", "readme", "requires-python", "license", - "license-files", "dependencies", "classifiers", "urls", "scripts", -}) - def package_version(stamps: list[int]) -> str: """The version of install.py from its commit times, newest first (UTC).""" @@ -85,36 +73,6 @@ def install_py_commits() -> list[int]: return [int(line) for line in out.split()] -def load_project() -> dict: - import tomllib # noqa: PLC0415 # 3.11, only where a package is built - - with (PACKAGE_DIR / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - unknown = set(project) - PROJECT_KEYS - if unknown: - raise ValueError(f"build_pypi.py writes no metadata for {sorted(unknown)}") - if project["dependencies"] or project["dynamic"] != ["version"]: - raise ValueError("the package has no dependencies and only its version is dynamic") - return project - - -def metadata(project: dict, version: str, readme: str) -> str: - """Core metadata 2.4, as the wheel's METADATA and the sdist's PKG-INFO.""" - lines = [ - "Metadata-Version: 2.4", - f"Name: {project['name']}", - f"Version: {version}", - f"Summary: {project['description']}", - *(f"Project-URL: {label}, {url}" for label, url in project["urls"].items()), - f"License-Expression: {project['license']}", - *(f"License-File: {name}" for name in project["license-files"]), - *(f"Classifier: {name}" for name in project["classifiers"]), - f"Requires-Python: {project['requires-python']}", - "Description-Content-Type: text/markdown", - ] - return "\n".join(lines) + "\n\n" + readme - - def installer_bytes() -> bytes: """install.py, refused unless it is the one install.sh pins: the package then runs the installer the one-line command runs, under the same id.""" @@ -125,87 +83,41 @@ def installer_bytes() -> bytes: return data -def package_files(version: str) -> tuple[dict, dict[str, bytes]]: - """The project table, and the files both archives ship, by path.""" - project = load_project() - readme = (PACKAGE_DIR / project["readme"]).read_text(encoding="utf-8") - files = { - f"{MODULE}/__init__.py": LAUNCHER.format(version=version).encode(), - f"{MODULE}/install.py": installer_bytes(), - "METADATA": metadata(project, version, readme).encode("utf-8"), - "README.md": readme.encode("utf-8"), - "pyproject.toml": (PACKAGE_DIR / "pyproject.toml").read_bytes(), - } - for name in project["license-files"]: - files[name] = (ROOT / name).read_bytes() - return project, files - - -def _record_line(path: str, data: bytes) -> str: - digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() - return f"{path},sha256={digest},{len(data)}" - - -def write_wheel(path: Path, project: dict, files: dict[str, bytes], version: str, - stamp: tuple[int, int, int]) -> None: - info = f"{project['name']}-{version}.dist-info" - scripts = "".join(f"{name} = {target}\n" for name, target in project["scripts"].items()) - members = { - f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], - f"{MODULE}/install.py": files[f"{MODULE}/install.py"], - f"{info}/METADATA": files["METADATA"], - f"{info}/WHEEL": (b"Wheel-Version: 1.0\nGenerator: retrobios build_pypi.py\n" - b"Root-Is-Purelib: true\nTag: py3-none-any\n"), - f"{info}/entry_points.txt": f"[console_scripts]\n{scripts}".encode(), - **{f"{info}/licenses/{name}": files[name] for name in project["license-files"]}, - } - record = "".join(_record_line(name, data) + "\n" for name, data in members.items()) - members[f"{info}/RECORD"] = (record + f"{info}/RECORD,,\n").encode() - with zipfile.ZipFile(path, "w") as archive: - for name, data in members.items(): - entry = zipfile.ZipInfo(name, date_time=(*stamp, 0, 0, 0)) - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = 0o644 << 16 - archive.writestr(entry, data) - - -def write_sdist(path: Path, project: dict, files: dict[str, bytes], version: str, - stamp: tuple[int, int, int]) -> None: - top = f"{project['name']}-{version}" - members = { - "PKG-INFO": files["METADATA"], - "pyproject.toml": files["pyproject.toml"], - "README.md": files["README.md"], - **{name: files[name] for name in project["license-files"]}, - f"{MODULE}/__init__.py": files[f"{MODULE}/__init__.py"], - f"{MODULE}/install.py": files[f"{MODULE}/install.py"], - } - mtime = calendar.timegm((*stamp, 0, 0, 0)) - raw = io.BytesIO() - with tarfile.open(fileobj=raw, mode="w", format=tarfile.PAX_FORMAT) as archive: - for name, data in members.items(): - entry = tarfile.TarInfo(f"{top}/{name}") - entry.size, entry.mtime, entry.mode = len(data), mtime, 0o644 - archive.addfile(entry, io.BytesIO(data)) - with path.open("wb") as handle, gzip.GzipFile(fileobj=handle, mode="wb", filename="", - mtime=mtime) as packed: - packed.write(raw.getvalue()) +def stage(folder: Path, version: str) -> None: + """The project as the backend reads it.""" + module = folder / MODULE + module.mkdir(parents=True) + for name in ("pyproject.toml", "README.md"): + shutil.copyfile(PACKAGE_DIR / name, folder / name) + shutil.copyfile(ROOT / "LICENSE", folder / "LICENSE") + shutil.copyfile(PACKAGE_DIR / MODULE / "__main__.py", module / "__main__.py") + (module / "__init__.py").write_text(LAUNCHER.format(version=version), encoding="utf-8") + (module / "install.py").write_bytes(installer_bytes()) def build(output_dir: Path) -> tuple[Path, Path]: """The wheel and the sdist in output_dir, replacing earlier builds there.""" - version = package_version(install_py_commits()) - year, month, day = (int(part) for part in version.split(".")[:3]) - stamp = (year, month, day) - project, files = package_files(version) + stamps = install_py_commits() + version = package_version(stamps) output_dir.mkdir(parents=True, exist_ok=True) for pattern in ("*.whl", "*.tar.gz"): - for old in output_dir.glob(f"{project['name']}-{pattern}"): + for old in output_dir.glob(f"{MODULE}-{pattern}"): old.unlink() - wheel = output_dir / f"{project['name']}-{version}-py3-none-any.whl" - sdist = output_dir / f"{project['name']}-{version}.tar.gz" - write_wheel(wheel, project, files, version, stamp) - write_sdist(sdist, project, files, version, stamp) + scratch = ROOT / "tmp" + scratch.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pypi-stage-", dir=scratch) as name: + folder = Path(name) / "retrobios" + stage(folder, version) + # Without a flag, uv builds the sdist, then the wheel from that sdist: + # what a user building from the sdist gets is what is published. + subprocess.run( + ["uv", "build", "--quiet", "--out-dir", str(output_dir.resolve()), str(folder)], + check=True, env={**os.environ, "SOURCE_DATE_EPOCH": str(stamps[0])}) + wheel = output_dir / f"{MODULE}-{version}-py3-none-any.whl" + sdist = output_dir / f"{MODULE}-{version}.tar.gz" + for path in (wheel, sdist): + if not path.is_file(): + raise RuntimeError(f"uv build wrote no {path.name}") return wheel, sdist diff --git a/tests/test_pypi_package.py b/tests/test_pypi_package.py index 812468b3e..4e49f5f69 100644 --- a/tests/test_pypi_package.py +++ b/tests/test_pypi_package.py @@ -2,16 +2,17 @@ `uvx retrobios`, `pipx run retrobios` and `pip install retrobios` reach the installer the one-line command runs: the same bytes, so the same id, the -same file lists, cache and exit codes. scripts/build_pypi.py writes the -wheel and the sdist, and these tests read what it wrote and run it. +same file lists, cache and exit codes. scripts/build_pypi.py stages the +project and builds it with `uv build` and the flit_core backend, and these +tests read what it wrote and run it. """ from __future__ import annotations import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache +import configparser import email.parser -import importlib.util import os import shutil import subprocess @@ -19,7 +20,6 @@ import sys import tarfile import tempfile import unittest -import venv import zipfile from pathlib import Path @@ -28,7 +28,7 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import build_pypi # noqa: E402 -MODULE_FILES = {"retrobios/__init__.py", "retrobios/install.py"} +MODULE_FILES = {"retrobios/__init__.py", "retrobios/__main__.py", "retrobios/install.py"} class Built(unittest.TestCase): @@ -42,6 +42,10 @@ class Built(unittest.TestCase): @classmethod def setUpClass(cls) -> None: + if shutil.which("uv") is None: + if os.environ.get("CI"): + raise AssertionError("CI installs uv") + raise unittest.SkipTest("uv is not installed") cls._scratch = tempfile.TemporaryDirectory(prefix="pypi-") cls.scratch = Path(cls._scratch.name) cls.wheel, cls.sdist = build_pypi.build(cls.scratch / "dist") @@ -65,13 +69,38 @@ class TheArchivesHoldTheInstallerAndNothingElse(Built): (REPO_ROOT / "install.py").read_bytes()) entry_points = archive.read(f"{info}/entry_points.txt").decode() meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) - self.assertEqual(entry_points, "[console_scripts]\nretrobios = retrobios:main\n") + scripts = configparser.ConfigParser() + scripts.read_string(entry_points) + self.assertEqual(dict(scripts["console_scripts"]), {"retrobios": "retrobios:main"}) self.assertEqual(meta["Name"], "retrobios") self.assertEqual(meta["Version"], self.version) self.assertEqual(meta["Requires-Python"], ">=3.8") self.assertIsNone(meta["Requires-Dist"]) - self.assertEqual(meta.get_payload(), - (REPO_ROOT / "packaging/pypi/README.md").read_text(encoding="utf-8")) + # The backend ends the body with one more newline than the file. + self.assertEqual(meta.get_payload().rstrip("\n"), + (REPO_ROOT / "packaging/pypi/README.md").read_text( + encoding="utf-8").rstrip("\n")) + + def test_metadata(self): + info = f"retrobios-{self.version}.dist-info" + with zipfile.ZipFile(self.wheel) as archive: + meta = email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) + self.assertEqual(meta["Metadata-Version"], "2.4") + self.assertEqual(meta["License-Expression"], "MIT") + classifiers = meta.get_all("Classifier") + # PEP 639: the license is the expression, never a classifier as well. + self.assertEqual([name for name in classifiers if name.startswith("License ::")], []) + self.assertEqual( + [name for name in classifiers if name.startswith("Programming Language :: Python :: 3.")], + [f"Programming Language :: Python :: 3.{minor}" for minor in range(8, 15)]) + for name in ("Programming Language :: Python :: 3 :: Only", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: End Users/Desktop", "Environment :: Console", + "Operating System :: OS Independent", "Topic :: System :: Emulators"): + self.assertIn(name, classifiers) + self.assertTrue(meta["Keywords"]) + self.assertEqual(sorted(url.split(",")[0] for url in meta.get_all("Project-URL")), + ["Documentation", "Homepage", "Issues", "Releases", "Source"]) def test_sdist_members(self): top = f"retrobios-{self.version}" @@ -91,6 +120,8 @@ class TheArchivesHoldTheInstallerAndNothingElse(Built): self.assertEqual([name for name in names if name.startswith(top)], [], top) def test_build_is_reproducible(self): + # SOURCE_DATE_EPOCH is install.py's commit time, so the clock moving + # between the two builds changes no byte. again = build_pypi.build(self.scratch / "again") self.assertEqual([path.read_bytes() for path in again], [self.wheel.read_bytes(), self.sdist.read_bytes()]) @@ -113,6 +144,23 @@ class TheVersionIsInstallPyCommitDate(unittest.TestCase): build_pypi.package_version([]) +class AnUnpinnedInstallerIsRefused(unittest.TestCase): + """The package runs the installer the one-line command runs, or none.""" + + def test_pin_mismatch(self): + with tempfile.TemporaryDirectory(prefix="pypi-pin-") as name: + root = Path(name) + (root / "install.py").write_bytes(b"print('another')\n") + shutil.copyfile(REPO_ROOT / "install.sh", root / "install.sh") + original = build_pypi.ROOT + build_pypi.ROOT = root + try: + with self.assertRaises(ValueError): + build_pypi.installer_bytes() + finally: + build_pypi.ROOT = original + + def _python38() -> str | None: found = shutil.which("python3.8") if found is None and os.environ.get("CI"): @@ -138,16 +186,19 @@ class TheInstalledCommandIsTheInstaller(Built): def _same_answers(self, python: str, name: str) -> None: command = self._installed(python, name) - for args in (["--help"], ["--version"], ["--no-such-option"]): - with self.subTest(python=name, args=args): - packaged = subprocess.run([str(command), *args], capture_output=True, - text=True, check=False, cwd=self.scratch) - script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], - capture_output=True, text=True, check=False, - cwd=self.scratch) - self.assertEqual( - (packaged.returncode, packaged.stdout, packaged.stderr), - (script.returncode, script.stdout, script.stderr)) + # `retrobios` and `python -m retrobios` both run install.py. + launchers = ([str(command)], [str(command.with_name("python")), "-m", "retrobios"]) + for launcher in launchers: + for args in (["--help"], ["--version"], ["--no-such-option"]): + with self.subTest(python=name, launcher=launcher[-1], args=args): + packaged = subprocess.run([*launcher, *args], capture_output=True, + text=True, check=False, cwd=self.scratch) + script = subprocess.run([python, str(REPO_ROOT / "install.py"), *args], + capture_output=True, text=True, check=False, + cwd=self.scratch) + self.assertEqual( + (packaged.returncode, packaged.stdout, packaged.stderr), + (script.returncode, script.stdout, script.stderr)) def test_this_python(self): self._same_answers(sys.executable, "venv-current") @@ -159,48 +210,5 @@ class TheInstalledCommandIsTheInstaller(Built): self._same_answers(python, "venv-38") -def _setuptools_builds_the_sdist() -> bool: - """setuptools 77 reads the license expression the sdist declares.""" - if importlib.util.find_spec("setuptools") is None: - return False - import setuptools # noqa: PLC0415 - - return int(setuptools.__version__.split(".")[0]) >= 77 - - -class TheSdistBuildsTheSameWheel(Built): - """pip and uv build the sdist with setuptools: the files must be those the - wheel written here holds.""" - - def test_setuptools_wheel(self): - if not _setuptools_builds_the_sdist(): - if os.environ.get("CI"): - self.fail("CI installs setuptools 77 or newer") - self.skipTest("setuptools 77 or newer is not installed") - out = self.scratch / "from-sdist" - work = self.scratch / "unpacked" - with tarfile.open(self.sdist) as archive: - archive.extractall(work, filter="data") - done = subprocess.run( - [sys.executable, "-m", "pip", "wheel", "--no-build-isolation", "--no-deps", - "--no-index", "--disable-pip-version-check", "-w", str(out), - str(work / f"retrobios-{self.version}")], - capture_output=True, text=True, check=False) - self.assertEqual(done.returncode, 0, done.stderr[-2000:]) - (built,) = out.glob("*.whl") - info = f"retrobios-{self.version}.dist-info" - with zipfile.ZipFile(built) as theirs, zipfile.ZipFile(self.wheel) as ours: - self.assertEqual(set(theirs.namelist()) - {f"{info}/top_level.txt"}, - set(ours.namelist())) - for name in (*MODULE_FILES, f"{info}/entry_points.txt"): - self.assertEqual(theirs.read(name), ours.read(name), name) - fields = ("Name", "Version", "Summary", "License-Expression", "Requires-Python") - theirs_meta, ours_meta = ( - email.parser.Parser().parsestr(archive.read(f"{info}/METADATA").decode()) - for archive in (theirs, ours)) - for field in fields: - self.assertEqual(theirs_meta[field], ours_meta[field], field) - - if __name__ == "__main__": unittest.main() diff --git a/wiki/release-process.md b/wiki/release-process.md index 0914e4a71..6b935e380 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -318,7 +318,9 @@ git commit -m "chore: record release v${DATE}" && git push # so an unchanged install.py keeps its version and PyPI refuses it # again. Publishing is not tied to a release: any push that changes # install.py can be published. The build replaces any earlier one in -# tmp/pypi-dist. Build, run the wheel, then upload the two files. The +# tmp/pypi-dist. `uv build` builds it with the flit_core backend the +# project pins, dated by that commit (SOURCE_DATE_EPOCH), so a rebuild +# gives the same bytes. Build, run the wheel, then upload the two files. The # token stays in its file: never in the repository, a script or a log. python scripts/build_pypi.py --output-dir tmp/pypi-dist python -m unittest tests.test_pypi_package