feat: watch the game catalogs for new DATs

automation/jobs.yml

@@ -77,12 +77,18 @@ jobs:
           Snapshots refreshed by refresh_stale from the MAME and FBNeo DATs
           check_freshness reported stale, behind a green offline pipeline.
         paths: [recipes/*.json]
+      - subject: "chore: refresh game catalogs"
+        body: >-
+          Catalogs imported again by refresh_stale from the No-Intro and
+          Redump DATs check_freshness reported stale, behind a green offline
+          pipeline.
+        paths: [games/catalogs/*/*.json]
       - subject: "chore: regenerate collection records"
         body: >-
           Regenerated by the offline pipeline, database.json and the
           manifests computed twice with the same result.
-        paths: [database.json, install/*.json, install/targets/*.json, README.md,
-                CONTRIBUTING.md, website/src/data/*.json]
+        paths: [database.json, install/*.json, install/targets/*.json, install/games/*.json,
+                install/games/*/*.json, README.md, CONTRIBUTING.md, website/src/data/*.json]
     scratch: [dist, tmp/autorun]
 
   bios-integrity:

scripts/check_freshness.py

@@ -40,6 +40,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import urllib.request
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import asdict, dataclass, field
@@ -62,6 +63,7 @@ from common import (  # noqa: E402
 )
 from exporter import discover_exporters  # noqa: E402
 from exporter.baseline import build_native_model  # noqa: E402
+from scripts.scraper import game_dat_importer  # noqa: E402
 from scripts.scraper.redump_dat_scraper import fetch_snapshot  # noqa: E402
 from scripts.scraper.targets import discover_target_scrapers  # noqa: E402
 
@@ -934,9 +936,87 @@ def check_catalogs(cache_dir: str, offline: bool) -> list[Finding]:
     findings.append(_check_tosec(offline))
     findings.append(_check_mame(cache_dir, offline))
     findings.append(_check_fbneo(cache_dir, offline))
+    findings.append(_check_nointro_games(cache_dir, offline))
+    findings.append(_check_redump_games(offline))
     return findings
 
 
+def _game_catalogs(source: str) -> list[dict]:
+    """The committed game catalogs of one source, games/catalogs/<source>/."""
+    return [_snapshot(path) for path in sorted((REPO_ROOT / "games" / "catalogs" / source).glob("*.json"))]
+
+
+def _no_intro_asset_date(cache_dir: str, offline: bool) -> str:
+    """The day the mirror last rebuilt its No-Intro pack, empty if unlisted."""
+    repo = upstream.parse_repo(NO_INTRO_MIRROR)
+    payload = upstream._api(  # noqa: SLF001
+        f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}",
+        cache_dir,
+        offline,
+        upstream.MOVING_TTL,
+    )
+    for asset in (payload.get("assets") if isinstance(payload, dict) else None) or []:
+        if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET:
+            return str(asset.get("updated_at", ""))[:10]
+    return ""
+
+
+def _check_nointro_games(cache_dir: str, offline: bool) -> Finding:
+    """games/catalogs/no-intro/ against the pack it was read from. The pack
+    holds every DAT: rebuilt after the catalogs were read, it may carry
+    games the games-on-card mode does not recognise."""
+    subject = "no-intro games"
+    fetched = min((str(doc.get("fetched", "")) for doc in _game_catalogs("no-intro")), default="")
+    if offline:
+        return Finding("catalogs", subject, SKIPPED, local=fetched, detail="offline")
+    updated = _no_intro_asset_date(cache_dir, offline)
+    if not updated:
+        return Finding("catalogs", subject, UNKNOWN, local=fetched, detail="asset not listed")
+    return Finding(
+        "catalogs", subject, STALE if updated > fetched else OK,
+        local=fetched, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}",
+    )
+
+
+def redump_datfile_version(code: str) -> str:
+    """The version redump.info serves for one datfile, read from the name it
+    gives the download: a HEAD request, nothing downloaded."""
+    url = game_dat_importer.REDUMP_DATFILE_URL.format(code=code)
+    request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "retrobios/1.0"})
+    with urllib.request.urlopen(request, timeout=60) as response:
+        disposition = response.headers.get("Content-Disposition", "")
+    named = game_dat_importer._DISPOSITION.search(disposition)  # noqa: SLF001
+    match = game_dat_importer._REDUMP_FILE.match(named["file"]) if named else None  # noqa: SLF001
+    if not match:
+        raise ValueError(f"{url} names no datfile")
+    return f"{match['date']} {match['time']}"
+
+
+def _check_redump_games(offline: bool) -> Finding:
+    """games/catalogs/redump/ against the datfiles redump.info serves now.
+    A disc dumped since is not recognised by the games-on-card mode."""
+    subject = "redump games"
+    local = {doc.get("system"): str(doc.get("version", "")) for doc in _game_catalogs("redump")}
+    newest = max(local.values(), default="")
+    if offline:
+        return Finding("catalogs", subject, SKIPPED, local=newest, detail="offline")
+    changed = []
+    remote_newest = ""
+    for code, (_name, system, _floor) in sorted(game_dat_importer.REDUMP_DATS.items()):
+        try:
+            remote = redump_datfile_version(code)
+        except (OSError, ValueError) as exc:
+            return Finding("catalogs", subject, UNKNOWN, local=newest, detail=str(exc))
+        remote_newest = max(remote_newest, remote)
+        if local.get(system) != remote:
+            changed.append(system)
+    return Finding(
+        "catalogs", subject, STALE if changed else OK, local=newest, remote=remote_newest,
+        detail=(f"changed: {', '.join(changed)}" if changed else f"{len(local)} DATs"),
+        items=changed,
+    )
+
+
 def _check_redump(offline: bool) -> Finding:
     local = _snapshot(REPO_ROOT / "provenance" / "redump.json")
     dats = local.get("dats") or {}
@@ -966,18 +1046,7 @@ def _check_no_intro(cache_dir: str, offline: bool) -> Finding:
     imported = str(local.get("imported_at", ""))
     if offline:
         return Finding("catalogs", "no-intro", SKIPPED, local=imported, detail="offline")
-    repo = upstream.parse_repo(NO_INTRO_MIRROR)
-    payload = upstream._api(  # noqa: SLF001
-        f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}",
-        cache_dir,
-        offline,
-        upstream.MOVING_TTL,
-    )
-    assets = payload.get("assets") if isinstance(payload, dict) else None
-    updated = ""
-    for asset in assets or []:
-        if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET:
-            updated = str(asset.get("updated_at", ""))[:10]
+    updated = _no_intro_asset_date(cache_dir, offline)
     if not updated:
         return Finding("catalogs", "no-intro", UNKNOWN, local=imported, detail="asset not listed")
     return Finding(

scripts/refresh_stale.py

@@ -416,6 +416,18 @@ def _command_for(
                 "fbneo",
                 "--fetch",
             ]
+        if subject in ("no-intro games", "redump games"):
+            # The game DATs download without a browser, unlike the BIOS ones.
+            return [
+                sys.executable,
+                "-m",
+                "scripts.scraper.game_dat_importer",
+                "--source",
+                subject.split()[0],
+                "--fetch",
+                "--refresh",
+                "--floor",
+            ]
         # redump / no-intro / tosec need manual DAT download.
         return None
 

tests/test_check_freshness.py

@@ -354,6 +354,25 @@ class PinParsingTests(unittest.TestCase):
 
 
 class CatalogParsingTests(unittest.TestCase):
+    def test_game_catalogs_are_watched(self):
+        """The 38 game catalogs the games-on-card mode identifies discs by
+        had no row: only the BIOS snapshots of provenance/ were compared."""
+        versions = {code: "2026-01-01 00-00-00" for code in cf.game_dat_importer.REDUMP_DATS}
+        versions["PSX"] = "2099-01-01 00-00-00"
+        local = [{"system": system, "version": "2026-01-01 00-00-00"}
+                 for _n, system, _f in cf.game_dat_importer.REDUMP_DATS.values()]
+        with mock.patch.object(cf, "_game_catalogs", return_value=local), \
+                mock.patch.object(cf, "redump_datfile_version", side_effect=versions.get):
+            row = cf._check_redump_games(False)
+            self.assertEqual((row.status, row.items), (cf.STALE, ["sony-playstation"]))
+            self.assertEqual(cf._check_redump_games(True).status, cf.SKIPPED)
+        with mock.patch.object(cf, "_game_catalogs", return_value=[{"fetched": "2026-11-03"}]):
+            for rebuilt, status in (("2026-07-08", cf.OK), ("2026-11-04", cf.STALE)):
+                with mock.patch.object(cf, "_no_intro_asset_date", return_value=rebuilt):
+                    self.assertEqual(cf._check_nointro_games("", False).status, status)
+        self.assertIn("no-intro games",
+                      [f.subject for f in cf.check_catalogs("", True)])
+
     def test_tosec_newest_release_from_category_links(self):
         html = (
             '<a href="/downloads/category/58-2024-05-17">x</a>'

tests/test_refresh_stale.py

@@ -120,6 +120,19 @@ class Planning(unittest.TestCase):
         )
         self.assertEqual([f["subject"] for f in surfaced], ["redump"])
 
+    def test_game_catalogs_are_imported_again(self):
+        jobs, _surfaced, _ = self._plan(
+            finding("catalogs", "no-intro games"), finding("catalogs", "redump games"))
+        self.assertEqual(
+            {j.subject: " ".join(j.command[1:]) for j in jobs},
+            {
+                "no-intro games": "-m scripts.scraper.game_dat_importer --source no-intro"
+                " --fetch --refresh --floor",
+                "redump games": "-m scripts.scraper.game_dat_importer --source redump"
+                " --fetch --refresh --floor",
+            },
+        )
+
     def test_a_data_row_outside_the_registry_needs_a_human(self):
         """refresh_data_dirs.py --key 'buildbot system' exits 1 on an unknown
         key: the row asks for a registry entry, not a refresh."""

wiki/tools.md

@@ -667,8 +667,9 @@ profile, `--record-coreinfo <core>` records its blob as reviewed. Data
 directories are checked against their upstream, dump catalogs and romset
 snapshots against the catalog they were read from (Redump DAT versions, the
 No-Intro daily rebuild, the newest TOSEC pack, the newest MAME release, the
-git blobs of the FBNeo DATs), and the CI toolchain against PyPI and the
-actions' latest releases.
+git blobs of the FBNeo DATs), the game catalogs of `games/catalogs/` against
+the No-Intro pack and the Redump datfiles they were read from, and the CI
+toolchain against PyPI and the actions' latest releases.
 
 The `native` area needs no network. It reads the cache `export_native.py`
 patches from and reports an original that is absent, that came from another
@@ -774,7 +775,7 @@ until a report without one. Exit codes: 0 for `ok`,
 
 | Job | Schedule | What it does |
 |-----|----------|--------------|
-| `nightly` | 02:00 | `check_freshness.py --json` saved to a file, `refresh_stale.py --findings` on its `data` and `catalogs` rows, the MAME and FBNeo hash scrapers, then `pipeline.py --offline` as the gate. `database.json` and `install/*.json` are computed a second time and compared. Commits the hash profiles, the recipes and the regenerated records |
+| `nightly` | 02:00 | `check_freshness.py --json` saved to a file, `refresh_stale.py --findings` on its `data` and `catalogs` rows, the MAME and FBNeo hash scrapers, then `pipeline.py --offline` as the gate. `database.json` and `install/*.json` are computed a second time and compared. Commits the hash profiles, the recipes, the game catalogs and the regenerated records |
 | `assets` | 04:30 | `sync_release_assets.py`, as a dry run |
 | `campaign` | 06:00 | `campaign.py run --budget 150m` once the host is idle (up to two hours of waiting), gated by `campaign.py gate`. Commits the records, findings and pass line, then the profile fields the runs proved |
 | `bios-integrity` | Sunday 08:30 | `bios_integrity.py --apply --json` in the main checkout, once the host is idle (up to four hours of waiting). Its report is kept in the job's status |