From c34ed4d2a10f0ccc363a68a51145b379d9c01ea1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:06:04 +0100 Subject: [PATCH] feat: watch the game catalogs for new DATs --- automation/jobs.yml | 10 +++- scripts/check_freshness.py | 93 ++++++++++++++++++++++++++++++----- scripts/refresh_stale.py | 12 +++++ tests/test_check_freshness.py | 19 +++++++ tests/test_refresh_stale.py | 13 +++++ wiki/tools.md | 7 +-- 6 files changed, 137 insertions(+), 17 deletions(-) diff --git a/automation/jobs.yml b/automation/jobs.yml index 1edfe158c..7ce719f0e 100644 --- a/automation/jobs.yml +++ b/automation/jobs.yml @@ -77,12 +77,18 @@ jobs: Snapshots refreshed by refresh_stale from the MAME and FBNeo DATs check_freshness reported stale, behind a green offline pipeline. paths: [recipes/*.json] + - subject: "chore: refresh game catalogs" + body: >- + Catalogs imported again by refresh_stale from the No-Intro and + Redump DATs check_freshness reported stale, behind a green offline + pipeline. + paths: [games/catalogs/*/*.json] - subject: "chore: regenerate collection records" body: >- Regenerated by the offline pipeline, database.json and the manifests computed twice with the same result. - paths: [database.json, install/*.json, install/targets/*.json, README.md, - CONTRIBUTING.md, website/src/data/*.json] + paths: [database.json, install/*.json, install/targets/*.json, install/games/*.json, + install/games/*/*.json, README.md, CONTRIBUTING.md, website/src/data/*.json] scratch: [dist, tmp/autorun] bios-integrity: diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index 57b77e65e..43e883dad 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -40,6 +40,7 @@ import shutil import subprocess import sys import tempfile +import urllib.request from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass, field @@ -62,6 +63,7 @@ from common import ( # noqa: E402 ) from exporter import discover_exporters # noqa: E402 from exporter.baseline import build_native_model # noqa: E402 +from scripts.scraper import game_dat_importer # noqa: E402 from scripts.scraper.redump_dat_scraper import fetch_snapshot # noqa: E402 from scripts.scraper.targets import discover_target_scrapers # noqa: E402 @@ -934,9 +936,87 @@ def check_catalogs(cache_dir: str, offline: bool) -> list[Finding]: findings.append(_check_tosec(offline)) findings.append(_check_mame(cache_dir, offline)) findings.append(_check_fbneo(cache_dir, offline)) + findings.append(_check_nointro_games(cache_dir, offline)) + findings.append(_check_redump_games(offline)) return findings +def _game_catalogs(source: str) -> list[dict]: + """The committed game catalogs of one source, games/catalogs//.""" + return [_snapshot(path) for path in sorted((REPO_ROOT / "games" / "catalogs" / source).glob("*.json"))] + + +def _no_intro_asset_date(cache_dir: str, offline: bool) -> str: + """The day the mirror last rebuilt its No-Intro pack, empty if unlisted.""" + repo = upstream.parse_repo(NO_INTRO_MIRROR) + payload = upstream._api( # noqa: SLF001 + f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}", + cache_dir, + offline, + upstream.MOVING_TTL, + ) + for asset in (payload.get("assets") if isinstance(payload, dict) else None) or []: + if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET: + return str(asset.get("updated_at", ""))[:10] + return "" + + +def _check_nointro_games(cache_dir: str, offline: bool) -> Finding: + """games/catalogs/no-intro/ against the pack it was read from. The pack + holds every DAT: rebuilt after the catalogs were read, it may carry + games the games-on-card mode does not recognise.""" + subject = "no-intro games" + fetched = min((str(doc.get("fetched", "")) for doc in _game_catalogs("no-intro")), default="") + if offline: + return Finding("catalogs", subject, SKIPPED, local=fetched, detail="offline") + updated = _no_intro_asset_date(cache_dir, offline) + if not updated: + return Finding("catalogs", subject, UNKNOWN, local=fetched, detail="asset not listed") + return Finding( + "catalogs", subject, STALE if updated > fetched else OK, + local=fetched, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}", + ) + + +def redump_datfile_version(code: str) -> str: + """The version redump.info serves for one datfile, read from the name it + gives the download: a HEAD request, nothing downloaded.""" + url = game_dat_importer.REDUMP_DATFILE_URL.format(code=code) + request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "retrobios/1.0"}) + with urllib.request.urlopen(request, timeout=60) as response: + disposition = response.headers.get("Content-Disposition", "") + named = game_dat_importer._DISPOSITION.search(disposition) # noqa: SLF001 + match = game_dat_importer._REDUMP_FILE.match(named["file"]) if named else None # noqa: SLF001 + if not match: + raise ValueError(f"{url} names no datfile") + return f"{match['date']} {match['time']}" + + +def _check_redump_games(offline: bool) -> Finding: + """games/catalogs/redump/ against the datfiles redump.info serves now. + A disc dumped since is not recognised by the games-on-card mode.""" + subject = "redump games" + local = {doc.get("system"): str(doc.get("version", "")) for doc in _game_catalogs("redump")} + newest = max(local.values(), default="") + if offline: + return Finding("catalogs", subject, SKIPPED, local=newest, detail="offline") + changed = [] + remote_newest = "" + for code, (_name, system, _floor) in sorted(game_dat_importer.REDUMP_DATS.items()): + try: + remote = redump_datfile_version(code) + except (OSError, ValueError) as exc: + return Finding("catalogs", subject, UNKNOWN, local=newest, detail=str(exc)) + remote_newest = max(remote_newest, remote) + if local.get(system) != remote: + changed.append(system) + return Finding( + "catalogs", subject, STALE if changed else OK, local=newest, remote=remote_newest, + detail=(f"changed: {', '.join(changed)}" if changed else f"{len(local)} DATs"), + items=changed, + ) + + def _check_redump(offline: bool) -> Finding: local = _snapshot(REPO_ROOT / "provenance" / "redump.json") dats = local.get("dats") or {} @@ -966,18 +1046,7 @@ def _check_no_intro(cache_dir: str, offline: bool) -> Finding: imported = str(local.get("imported_at", "")) if offline: return Finding("catalogs", "no-intro", SKIPPED, local=imported, detail="offline") - repo = upstream.parse_repo(NO_INTRO_MIRROR) - payload = upstream._api( # noqa: SLF001 - f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}", - cache_dir, - offline, - upstream.MOVING_TTL, - ) - assets = payload.get("assets") if isinstance(payload, dict) else None - updated = "" - for asset in assets or []: - if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET: - updated = str(asset.get("updated_at", ""))[:10] + updated = _no_intro_asset_date(cache_dir, offline) if not updated: return Finding("catalogs", "no-intro", UNKNOWN, local=imported, detail="asset not listed") return Finding( diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 27c9bda3d..4e79e70a5 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -416,6 +416,18 @@ def _command_for( "fbneo", "--fetch", ] + if subject in ("no-intro games", "redump games"): + # The game DATs download without a browser, unlike the BIOS ones. + return [ + sys.executable, + "-m", + "scripts.scraper.game_dat_importer", + "--source", + subject.split()[0], + "--fetch", + "--refresh", + "--floor", + ] # redump / no-intro / tosec need manual DAT download. return None diff --git a/tests/test_check_freshness.py b/tests/test_check_freshness.py index d1d58d421..5bbecd396 100644 --- a/tests/test_check_freshness.py +++ b/tests/test_check_freshness.py @@ -354,6 +354,25 @@ class PinParsingTests(unittest.TestCase): class CatalogParsingTests(unittest.TestCase): + def test_game_catalogs_are_watched(self): + """The 38 game catalogs the games-on-card mode identifies discs by + had no row: only the BIOS snapshots of provenance/ were compared.""" + versions = {code: "2026-01-01 00-00-00" for code in cf.game_dat_importer.REDUMP_DATS} + versions["PSX"] = "2099-01-01 00-00-00" + local = [{"system": system, "version": "2026-01-01 00-00-00"} + for _n, system, _f in cf.game_dat_importer.REDUMP_DATS.values()] + with mock.patch.object(cf, "_game_catalogs", return_value=local), \ + mock.patch.object(cf, "redump_datfile_version", side_effect=versions.get): + row = cf._check_redump_games(False) + self.assertEqual((row.status, row.items), (cf.STALE, ["sony-playstation"])) + self.assertEqual(cf._check_redump_games(True).status, cf.SKIPPED) + with mock.patch.object(cf, "_game_catalogs", return_value=[{"fetched": "2026-11-03"}]): + for rebuilt, status in (("2026-07-08", cf.OK), ("2026-11-04", cf.STALE)): + with mock.patch.object(cf, "_no_intro_asset_date", return_value=rebuilt): + self.assertEqual(cf._check_nointro_games("", False).status, status) + self.assertIn("no-intro games", + [f.subject for f in cf.check_catalogs("", True)]) + def test_tosec_newest_release_from_category_links(self): html = ( 'x' diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index db5a7dacd..3310d068e 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -120,6 +120,19 @@ class Planning(unittest.TestCase): ) self.assertEqual([f["subject"] for f in surfaced], ["redump"]) + def test_game_catalogs_are_imported_again(self): + jobs, _surfaced, _ = self._plan( + finding("catalogs", "no-intro games"), finding("catalogs", "redump games")) + self.assertEqual( + {j.subject: " ".join(j.command[1:]) for j in jobs}, + { + "no-intro games": "-m scripts.scraper.game_dat_importer --source no-intro" + " --fetch --refresh --floor", + "redump games": "-m scripts.scraper.game_dat_importer --source redump" + " --fetch --refresh --floor", + }, + ) + def test_a_data_row_outside_the_registry_needs_a_human(self): """refresh_data_dirs.py --key 'buildbot system' exits 1 on an unknown key: the row asks for a registry entry, not a refresh.""" diff --git a/wiki/tools.md b/wiki/tools.md index 31a0b756d..653fbac01 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -667,8 +667,9 @@ profile, `--record-coreinfo ` records its blob as reviewed. Data directories are checked against their upstream, dump catalogs and romset snapshots against the catalog they were read from (Redump DAT versions, the No-Intro daily rebuild, the newest TOSEC pack, the newest MAME release, the -git blobs of the FBNeo DATs), and the CI toolchain against PyPI and the -actions' latest releases. +git blobs of the FBNeo DATs), the game catalogs of `games/catalogs/` against +the No-Intro pack and the Redump datfiles they were read from, and the CI +toolchain against PyPI and the actions' latest releases. The `native` area needs no network. It reads the cache `export_native.py` patches from and reports an original that is absent, that came from another @@ -774,7 +775,7 @@ until a report without one. Exit codes: 0 for `ok`, | Job | Schedule | What it does | |-----|----------|--------------| -| `nightly` | 02:00 | `check_freshness.py --json` saved to a file, `refresh_stale.py --findings` on its `data` and `catalogs` rows, the MAME and FBNeo hash scrapers, then `pipeline.py --offline` as the gate. `database.json` and `install/*.json` are computed a second time and compared. Commits the hash profiles, the recipes and the regenerated records | +| `nightly` | 02:00 | `check_freshness.py --json` saved to a file, `refresh_stale.py --findings` on its `data` and `catalogs` rows, the MAME and FBNeo hash scrapers, then `pipeline.py --offline` as the gate. `database.json` and `install/*.json` are computed a second time and compared. Commits the hash profiles, the recipes, the game catalogs and the regenerated records | | `assets` | 04:30 | `sync_release_assets.py`, as a dry run | | `campaign` | 06:00 | `campaign.py run --budget 150m` once the host is idle (up to two hours of waiting), gated by `campaign.py gate`. Commits the records, findings and pass line, then the profile fields the runs proved | | `bios-integrity` | Sunday 08:30 | `bios_integrity.py --apply --json` in the main checkout, once the host is idle (up to four hours of waiting). Its report is kept in the job's status |