fix: an empty or HTML original fails the export

fetch checked archives only, so a System.dat served as an HTML page with 200, or a 0-byte cache, reached render() as a file with no entries. System.dat, es_bios.xml, RetroBat and RomM then rebuilt the platform's file from our data and reported OK. Such an original is now refused, before it is cached and when read from the cache, and a format that needs its original fails when none was read.
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 03:59:07 +01:00
1 parent d5e61308e1
commit 6427e46e25
2 files changed
+45 -2

No files matched your search

+23 -2
View File
@@ -105,6 +105,25 @@ def _read_archive(unpack: Callable[[bytes], dict[str, str]], payload: bytes) ->
) from exc
def _read_text(payload: bytes) -> str:
"""A text original, refused when it cannot be the platform's file.
An empty body or an HTML error page served with 200 parsed as a file
with no entries, and four exporters rebuilt the platform's file from our
data alone and said OK: libretro's quoted names and Recalbox's comments
were gone.
"""
text = payload.decode("utf-8", errors="replace")
head = text.lstrip()[:64].lower()
if not head:
raise OriginalUnreadable("empty file: --refresh-cache downloads it again")
if head.startswith(("<!doctype html", "<html")):
raise OriginalUnreadable(
"an HTML page, not the platform's file: --refresh-cache downloads it again"
)
return text
def fetch(
url: str,
destination: Path,
@@ -331,7 +350,7 @@ def collect_originals(
for relative, url in wanted.items():
path = root / relative
archive = callable(unpack) and relative.endswith((".zip", ".tar.gz"))
check = (lambda raw: _read_archive(unpack, raw)) if archive else None
check = (lambda raw: _read_archive(unpack, raw)) if archive else _read_text
try:
if allow_fetch:
payload = fetch(url, path, index, refresh, check)
@@ -348,7 +367,7 @@ def collect_originals(
if callable(keep):
keep(relative, payload)
else:
originals[relative] = payload.decode("utf-8", errors="replace")
originals[relative] = _read_text(payload)
# One platform's unreadable file fails that platform: raised, it
# stopped --all before the platforms after it.
except (OSError, http.client.HTTPException, OriginalUnreadable) as exc:
@@ -699,6 +718,8 @@ def export_platform(
)
if missing:
return False, [f"the platform's own file is required: {m}" for m in missing]
if exporter.needs_original() and not originals:
return False, ["the platform's own file is required and none was read"]
try:
produced = exporter.render(systems, report, originals, scraped)
+22
View File
@@ -1010,6 +1010,28 @@ class CachedOriginal(unittest.TestCase):
)
self.assertEqual((originals, missing), ({"list.txt": "two"}, []))
def test_an_empty_or_html_original_fails_the_platform(self):
"""A System.dat served as an HTML page with 200, or a 0-byte cache,
was parsed as a file with no entries: four exporters rebuilt it from
our data and reported OK."""
for body in ("", " \n", "<html><body>Too Many Requests</body></html>",
"<!DOCTYPE html>\n<html></html>"):
with self.subTest(body=body):
url = self._upstream("served.txt", body)
originals, missing = collect_originals(
_OneFileExporter(url), {}, self.cache, True, refresh=True
)
self.assertEqual(originals, {})
self.assertEqual(len(missing), 1)
cached = self.cache / "plat" / "list.txt"
cached.parent.mkdir(parents=True, exist_ok=True)
cached.write_text(body, encoding="utf-8")
_, missing = collect_originals(
_OneFileExporter(url), {}, self.cache, False
)
self.assertEqual(len(missing), 1)
cached.unlink()
def test_offline_serves_what_is_cached_and_names_what_is_not(self):
url = self._upstream("v1.txt", "one")
_, missing = collect_originals(_OneFileExporter(url), {}, self.cache, False)