From 49b79a6d5225dc8a7b66cdcd742fe31e39f033dd Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:40:28 +0200 Subject: [PATCH] fix: refuse a file read off the pinned revision pinned_base returned an empty base when the platform's source named no revision or no longer ended in the exported file, and the exporter's own branch URL was then patched without a word. Every file an export reads must now carry the revision the platform YAML records, or the platform fails and asks for a rescrape. The check covers every file of every platform: ROCKNIX's second file, RetroDECK's manifests and RetroPie's tarball were checked by nothing before. --- scripts/export_native.py | 41 ++++++++++++++++++++++++++++++++++++++++ tests/test_exporters.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/scripts/export_native.py b/scripts/export_native.py index b50c7ef5..32af276b 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -236,6 +236,41 @@ def wanted_sources( } +_REVISION = ( + re.compile(r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/"), + re.compile(r"^https://github\.com/[^/]+/[^/]+/(?:tree|blob)/([^/]+)"), + re.compile(r"^https://gitlab\.com/.+?/-/(?:raw|archive)/([^/]+)/"), + re.compile(r"^https://codeload\.github\.com/[^/]+/[^/]+/tar\.gz/([^/]+)$"), +) + + +def source_revision(source: str) -> str: + """The revision a platform YAML's source: URL names, '' when none.""" + for pattern in _REVISION: + match = pattern.match(source) + if match: + return match.group(1) + return "" + + +def unpinned(wanted: dict[str, str], scraped: dict | None) -> list[str]: + """Every file the export would read off the revision the YAML records. + + pinned_base() returned '' when the source named no revision or no + longer ended in the file, and the exporter's own branch URL was then + patched without a word: a file our transcription never described. + """ + source = str((scraped or {}).get("source", "")) + revision = source_revision(source) + if not revision: + return [f"the platform's source names no revision ({source or 'none'}): rescrape it"] + return [ + f"{relative} would be read at {url}, not at {revision}: rescrape the platform" + for relative, url in wanted.items() + if f"/{revision}/" not in url and not url.endswith(f"/{revision}") + ] + + _CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/") @@ -381,6 +416,9 @@ def refresh_cache( systems, _ = build_native_model(truth or {}, scraped) exporter = exporter_class() wanted = wanted_sources(exporter, systems, scraped) + off_pin = unpinned(wanted, scraped) + if off_pin: + return False, off_pin _, missing = collect_originals( exporter, systems, upstream_dir, True, scraped, refresh=True ) @@ -496,6 +534,9 @@ def export_platform( return False, ["nothing to write: neither the platform nor the truth has data"] exporter = exporter_class() + off_pin = unpinned(wanted_sources(exporter, systems, scraped), scraped) + if off_pin: + return False, off_pin originals, missing = collect_originals( exporter, systems, upstream_dir, allow_fetch, scraped ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b3e4dc8d..51161d8c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -34,6 +34,7 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) from common import ( # noqa: E402 list_registered_platforms, load_platform_config, + native_layer, parse_untrusted_xml, ) from exporter import discover_exporters # noqa: E402 @@ -697,6 +698,35 @@ class PinnedRevision(unittest.TestCase): base = pinned_base(dict(exporters[name].native_sources()), config) self.assertTrue(base.startswith("https://"), base) + def test_every_file_of_every_platform_is_read_at_the_pinned_revision(self): + """Checked on five platforms by their base only: ROCKNIX's second + file, RetroDECK's manifests and RetroPie's tarball were never + looked at, and a URL that fell back to a branch passed.""" + import export_native # noqa: PLC0415 + + for name, exporter_class in discover_exporters().items(): + with self.subTest(platform=name): + config = native_layer(load_platform_config(name, str(REPO_ROOT / "platforms"))) + systems, _ = build_native_model({}, config) + wanted = export_native.wanted_sources(exporter_class(), systems, config) + self.assertTrue(wanted) + self.assertEqual(export_native.unpinned(wanted, config), []) + + def test_a_file_read_off_the_pinned_revision_is_named(self): + import export_native # noqa: PLC0415 + + pinned = {"source": "https://raw.githubusercontent.com/o/r/v1.0/list.txt"} + self.assertEqual(export_native.unpinned( + {"list.txt": "https://raw.githubusercontent.com/o/r/v1.0/list.txt"}, pinned), []) + moved = export_native.unpinned( + {"list.txt": "https://raw.githubusercontent.com/o/r/master/list.txt"}, pinned) + self.assertEqual(len(moved), 1) + self.assertIn("v1.0", moved[0]) + bare = export_native.unpinned( + {"x.json": "https://raw.githubusercontent.com/o/r/main/x.json"}, + {"source": "https://github.com/RetroDECK/components"}) + self.assertIn("names no revision", bare[0]) + class _OneFileExporter: """An exporter reduced to what collect_originals reads."""