From 7080459011c1d51715618a47ed0716ebf4e1a8c3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:59:40 +0100 Subject: [PATCH] fix: read No-Intro from a mirror that rebuilds --- scripts/check_freshness.py | 43 ++++++++++++++++++---------- scripts/scraper/game_dat_importer.py | 10 +++++-- tests/test_check_freshness.py | 22 ++++++++++++-- wiki/tools.md | 2 +- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index f83c84588..ac1e02188 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -75,9 +75,13 @@ REPO_ROOT = Path(__file__).resolve().parents[1] CORE_INFO_REPO = "https://github.com/libretro/libretro-core-info" # The git blob of every .info as last reviewed: `--record-coreinfo` writes it. CORE_INFO_RECORD = REPO_ROOT / "platforms" / "_core_info.json" -NO_INTRO_MIRROR = "https://github.com/hugo19941994/auto-datfile-generator" -NO_INTRO_RELEASE = "Daily_Rebuild" -NO_INTRO_ASSET = "no-intro.zip" +NO_INTRO_MIRROR = game_dat_importer.NOINTRO_MIRROR +NO_INTRO_RELEASE = game_dat_importer.NOINTRO_RELEASE +NO_INTRO_ASSET = game_dat_importer.NOINTRO_ASSET +# The mirror rebuilds daily. A pack older than this is a mirror that stopped: +# hugo19941994's failed every day from 2026-07-08, and comparing our import +# date with its asset read OK for four months. +NO_INTRO_MAX_AGE_DAYS = 7 TOSEC_DOWNLOADS = "https://www.tosecdev.org/downloads" MAME_REPO = "https://github.com/mamedev/mame" FBNEO_REPO = "https://github.com/libretro/FBNeo" @@ -1088,6 +1092,25 @@ def _no_intro_asset_date(cache_dir: str, offline: bool) -> str: return "" +def _today() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def _no_intro_row(subject: str, local: str, updated: str) -> Finding: + """Our import against the mirror's pack, and the pack against today.""" + age = (datetime.fromisoformat(_today()) - datetime.fromisoformat(updated)).days + if age > NO_INTRO_MAX_AGE_DAYS: + return Finding( + "catalogs", subject, STALE, local=local, remote=updated, + detail=f"the mirror last rebuilt {NO_INTRO_ASSET} on {updated}, {age} days ago, " + f"though it rebuilds daily: {NO_INTRO_MIRROR}/actions", + ) + return Finding( + "catalogs", subject, STALE if updated > local else OK, + local=local, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}", + ) + + def _check_nointro_games(cache_dir: str, offline: bool) -> Finding: """games/catalogs/no-intro/ against the pack it was read from. The pack holds every DAT: rebuilt after the catalogs were read, it may carry @@ -1099,10 +1122,7 @@ def _check_nointro_games(cache_dir: str, offline: bool) -> Finding: updated = _no_intro_asset_date(cache_dir, offline) if not updated: return Finding("catalogs", subject, UNKNOWN, local=fetched, detail="asset not listed") - return Finding( - "catalogs", subject, STALE if updated > fetched else OK, - local=fetched, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}", - ) + return _no_intro_row(subject, fetched, updated) def redump_datfile_version(code: str) -> str: @@ -1176,14 +1196,7 @@ def _check_no_intro(cache_dir: str, offline: bool) -> Finding: updated = _no_intro_asset_date(cache_dir, offline) if not updated: return Finding("catalogs", "no-intro", UNKNOWN, local=imported, detail="asset not listed") - return Finding( - "catalogs", - "no-intro", - STALE if updated > imported else OK, - local=imported, - remote=updated, - detail=f"{NO_INTRO_ASSET} rebuilt {updated}", - ) + return _no_intro_row("no-intro", imported, updated) def tosec_latest_pack(html: str) -> str | None: diff --git a/scripts/scraper/game_dat_importer.py b/scripts/scraper/game_dat_importer.py index 0968008bf..87522041c 100644 --- a/scripts/scraper/game_dat_importer.py +++ b/scripts/scraper/game_dat_importer.py @@ -14,6 +14,9 @@ games/catalogs/ and never in provenance/. No-Intro comes from the auto-datfile-generator release asset (Dat-o-Matic bans automated clients), Redump from redump.info's per-system datfile route. +The mirror is dantob's fork: hugo19941994's original has failed its daily +rebuild since Dat-o-Matic's 2026-07 re-layout (no-intro.zip last rebuilt +2026-07-08), and the fork carries the fix (d4a9d769, 2026-07-31). Every download is cached under .cache/game-dats/ with what the server said about it, so a second run reads nothing from the network and writes the same bytes. A DAT's region tag is a market label, never the value a core computes. @@ -49,8 +52,11 @@ SOURCES = ("no-intro", "redump") CACHE_DIR = Path(".cache") / "game-dats" OUTPUT_DIR = Path("games") / "catalogs" +NOINTRO_MIRROR = "https://github.com/dantob/auto-datfile-generator" +NOINTRO_RELEASE = "Daily_Rebuild" NOINTRO_RELEASE_API = ( - "https://api.github.com/repos/hugo19941994/auto-datfile-generator/releases/tags/Daily_Rebuild" + NOINTRO_MIRROR.replace("https://github.com/", "https://api.github.com/repos/") + + f"/releases/tags/{NOINTRO_RELEASE}" ) NOINTRO_ASSET = "no-intro.zip" REDUMP_DATFILE_URL = "https://redump.info/datfile/{code}" @@ -120,7 +126,7 @@ LICENCES: dict[str, dict] = { "website or its operators is required for such use." ), "mirror": { - "repo": "https://github.com/hugo19941994/auto-datfile-generator", + "repo": NOINTRO_MIRROR, "licence": "MIT", "scope": "the mirror's scripts only, the DATs it republishes stay under the terms above", }, diff --git a/tests/test_check_freshness.py b/tests/test_check_freshness.py index a7bd3d078..53853909d 100644 --- a/tests/test_check_freshness.py +++ b/tests/test_check_freshness.py @@ -475,10 +475,28 @@ class CatalogParsingTests(unittest.TestCase): row = cf._check_redump_games(False) self.assertEqual((row.status, row.items), (cf.STALE, ["sony-playstation"])) self.assertEqual(cf._check_redump_games(True).status, cf.SKIPPED) - with mock.patch.object(cf, "_game_catalogs", return_value=[{"fetched": "2026-11-03"}]): - for rebuilt, status in (("2026-07-08", cf.OK), ("2026-11-04", cf.STALE)): + with mock.patch.object(cf, "_game_catalogs", return_value=[{"fetched": "2026-11-03"}]), \ + mock.patch.object(cf, "_today", return_value="2026-11-06"): + for rebuilt, status in (("2026-11-02", cf.OK), ("2026-11-04", cf.STALE)): with mock.patch.object(cf, "_no_intro_asset_date", return_value=rebuilt): self.assertEqual(cf._check_nointro_games("", False).status, status) + + def test_a_mirror_that_stopped_rebuilding_is_stale(self): + """The mirror failed every daily rebuild from 2026-07-08, and both + No-Intro rows read OK: an import newer than a dead pack is no + sign of a fresh catalog.""" + with mock.patch.object(cf, "_today", return_value="2026-11-06"), \ + mock.patch.object(cf, "_no_intro_asset_date", return_value="2026-07-08"), \ + mock.patch.object(cf, "_game_catalogs", return_value=[{"fetched": "2026-11-03"}]), \ + mock.patch.object(cf, "_snapshot", return_value={"imported_at": "2026-08-08"}): + for row in (cf._check_nointro_games("", False), cf._check_no_intro("", False)): + self.assertEqual(row.status, cf.STALE) + self.assertIn("2026-07-08, 121 days ago", row.detail) + + def test_the_freshness_check_reads_the_importer_mirror(self): + self.assertEqual(cf.NO_INTRO_MIRROR, cf.game_dat_importer.NOINTRO_MIRROR) + self.assertIn(cf.NO_INTRO_MIRROR.removeprefix("https://github.com/"), + cf.game_dat_importer.NOINTRO_RELEASE_API) self.assertIn("no-intro games", [f.subject for f in cf.check_catalogs("", True)]) diff --git a/wiki/tools.md b/wiki/tools.md index eb35ae354..221f8a28d 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1659,7 +1659,7 @@ names the profiles read. | Catalog | Acquisition | Notes | |---------|------------|-------| | Redump | direct fetch from redump.info, one `/bios/` route per system | 4 BIOS DATs. redump.org serves BIOS DATs dated 2010 to 2022, redump.info the current ones | -| No-Intro | local pack, imported with `dat_pack_importer` | Dat-o-Matic serves its DATs through a browser session, and the `hugo19941994/auto-datfile-generator` mirror rebuilds daily | +| No-Intro | local pack, imported with `dat_pack_importer` | Dat-o-Matic serves its DATs through a browser session, and the `dantob/auto-datfile-generator` mirror rebuilds daily. `check_freshness` reads a pack older than a week as a stopped mirror | | TOSEC | local pack, imported with `dat_pack_importer` | annual pack from tosecdev.org, 138 Firmware DATs | The join is by SHA1, then by MD5 + size. A match sets the `provenance` field