diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index b8e03f28b..3f703d768 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -509,7 +509,10 @@ def check_native( ) -> list[Finding]: exporters = discover_exporters() index = cache_dir / export_native.SOURCES_INDEX - recorded = export_native.load_sources(cache_dir) + try: + recorded = export_native.load_sources(cache_dir) + except export_native.SourcesIndexUnreadable as exc: + return [Finding("native", index.name, ERROR, detail=str(exc))] findings: list[Finding] = [] for name in list_registered_platforms(str(platforms_dir), include_archived=True): exporter_class = exporters.get(name) diff --git a/scripts/export_native.py b/scripts/export_native.py index 42d529eba..d91e9e783 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex import yaml from common import ( + InputMissing, build_zip_contents_index, platform_arg, list_registered_platforms, @@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0" _MAX_BYTES = 64 * 1024 * 1024 +class SourcesIndexUnreadable(InputMissing): + """The cache's URL index exists and does not read as JSON.""" + + def _load_sources(index: Path | None) -> dict[str, str]: + """The URL each cached file was fetched from. + + An absent index records nothing. An index that does not read raises: + read as empty it let every cached original pass for current, and the + next download rewrote it with that one entry. + """ if index is None or not index.is_file(): return {} try: return json.loads(index.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} + except (OSError, json.JSONDecodeError) as exc: + raise SourcesIndexUnreadable( + f"{index} does not read ({exc}): remove it, then" + " python scripts/export_native.py --all --refresh-cache" + " downloads every original again and records its URL" + ) from exc def source_key(destination: Path, index: Path | None) -> str: diff --git a/tests/test_check_freshness.py b/tests/test_check_freshness.py index 72846872f..9fedcc590 100644 --- a/tests/test_check_freshness.py +++ b/tests/test_check_freshness.py @@ -35,6 +35,14 @@ class NativeCacheTests(unittest.TestCase): self.root.mkdir() self.wanted = {"list.txt": "https://example.invalid/tag-2/list.txt"} + def test_an_unreadable_index_is_an_error_not_unrecorded(self): + """Read as empty, it named "no URL recorded" for every original.""" + self.index.write_text("{", encoding="utf-8") + findings = cf.check_native(REPO_ROOT / "platforms", self.cache, self.cache) + self.assertEqual([(f.subject, f.status) for f in findings], + [(self.index.name, cf.ERROR)]) + self.assertIn("--refresh-cache", findings[0].detail) + def _cached(self, mtime: float) -> None: path = self.root / "list.txt" path.write_text("x", encoding="utf-8") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f84b4e624..391171064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1005,6 +1005,22 @@ class CachedOriginal(unittest.TestCase): self._upstream("v1.txt", "moved") self.assertEqual(fetch(url, target, self.index), b"one") + def test_an_unreadable_index_serves_nothing(self): + """A truncated index read as empty: the cached batocera-41 file was + served for a 43.1 URL, and the next download erased every other + platform's recorded URL.""" + import export_native + + old = self._upstream("batocera-41.txt", "old") + target = self.cache / "plat" / "list.txt" + fetch(old, target, self.index) + self.index.write_text('{"plat/list.txt": "' + old + '",', encoding="utf-8") + new = self._upstream("batocera-43.txt", "new") + with self.assertRaises(export_native.SourcesIndexUnreadable) as caught: + fetch(new, target, self.index) + self.assertIn("--refresh-cache", str(caught.exception)) + self.assertTrue(self.index.read_text(encoding="utf-8").endswith(",")) + def test_refresh_reads_a_branch_url_again(self): """A branch URL keeps its name while the platform moves.""" url = self._upstream("branch.txt", "one")