feat: keep cached platform originals in step

scripts/check_freshness.py

@@ -12,6 +12,11 @@ into a throwaway copy, then diffed against the committed file. The diff
 decides, not a version string: a scraper that pins a new tag but produces
 the same entries is reported as a version change and nothing else.
 
+The native export patches each platform's own file, kept in a local cache.
+That cache is checked against the platform file it was transcribed with: an
+original cached before the last rescrape, or under another pin, is no longer
+the file our data describes.
+
 Emulator profiles are covered by profile_sync, which is slow (one API pass
 per profile). ``--profiles`` runs it here and folds its verdict in.
 
@@ -43,6 +48,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
+import export_native  # noqa: E402
 import refresh_data_dirs  # noqa: E402
 import upstream  # noqa: E402
 from common import (  # noqa: E402
@@ -50,6 +56,8 @@ from common import (  # noqa: E402
     load_emulator_profiles,
     yaml_load,
 )
+from exporter import discover_exporters  # noqa: E402
+from exporter.baseline import build_native_model  # noqa: E402
 from scripts.scraper.redump_dat_scraper import fetch_snapshot  # noqa: E402
 from scripts.scraper.targets import discover_target_scrapers  # noqa: E402
 
@@ -66,7 +74,7 @@ INSTALLERS = ("install.sh", "install.ps1")
 
 SHOWN_ITEMS = 4  # items named in a diff summary before ", ..."
 SHOWN_NAMES = 12  # core names named in an unresolved row before ", ..."
-AREAS = ("platforms", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
+AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
 OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED"
 # Every status a finding may carry, in the order the summary counts them.
 STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK)
@@ -279,6 +287,80 @@ def check_platforms(platforms_dir: Path, workdir: Path, jobs: int) -> list[Findi
     return sorted(findings, key=lambda f: f.subject)
 
 
+# --- native originals --------------------------------------------------------
+
+
+def native_cache_state(
+    wanted: dict[str, str],
+    root: Path,
+    recorded: dict[str, str],
+    index: Path,
+    transcribed_at: float,
+) -> str:
+    """Why a platform's cached originals cannot be patched, '' when they can.
+
+    The export corrects the file our data was transcribed from. A cached
+    original is that file only if it came from the URL the platform file
+    names today and was fetched no earlier than the platform file was last
+    rewritten: a branch URL keeps its name while its content moves.
+    """
+    for relative, url in sorted(wanted.items()):
+        path = root / relative
+        if not path.is_file():
+            return f"{relative} not cached"
+        known = recorded.get(export_native.source_key(path, index))
+        if known is None:
+            return f"{relative} cached from an unrecorded URL"
+        if known != url:
+            return f"{relative} cached from another revision"
+        if path.stat().st_mtime < transcribed_at:
+            return f"{relative} cached before the platform file was rewritten"
+    return ""
+
+
+def _transcribed_at(platforms_dir: Path, platform: str) -> float:
+    """Last rewrite of the platform file or of any file it inherits."""
+    newest = 0.0
+    seen: set[str] = set()
+    name: str | None = platform
+    while name and name not in seen:
+        seen.add(name)
+        path = platforms_dir / f"{name}.yml"
+        if not path.is_file():
+            break
+        newest = max(newest, path.stat().st_mtime)
+        with path.open(encoding="utf-8") as fh:
+            name = (yaml_load(fh) or {}).get("inherits")
+    return newest
+
+
+def check_native(platforms_dir: Path, cache_dir: Path, truth_dir: Path) -> list[Finding]:
+    exporters = discover_exporters()
+    index = cache_dir / export_native.SOURCES_INDEX
+    recorded = export_native.load_sources(cache_dir)
+    findings: list[Finding] = []
+    for name in list_registered_platforms(str(platforms_dir), include_archived=True):
+        exporter_class = exporters.get(name)
+        if not exporter_class:
+            continue
+        truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir))
+        systems, _ = build_native_model(truth or {}, scraped)
+        wanted = export_native.wanted_sources(exporter_class(), systems, scraped)
+        reason = native_cache_state(
+            wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
+        )
+        findings.append(
+            Finding(
+                "native",
+                name,
+                STALE if reason else OK,
+                local=f"{len(wanted)} file(s)",
+                detail=f"{reason}, export_native.py --refresh-cache fetches it" if reason else "",
+            )
+        )
+    return sorted(findings, key=lambda f: f.subject)
+
+
 # --- targets ----------------------------------------------------------------
 
 
@@ -959,6 +1041,8 @@ def main() -> int:
     parser.add_argument("--offline", action="store_true", help="no network, local ages only")
     parser.add_argument("--jobs", type=int, default=4, help="parallel scrapers")
     parser.add_argument("--cache-dir", default=".cache/upstream")
+    parser.add_argument("--native-cache-dir", default=export_native.DEFAULT_CACHE)
+    parser.add_argument("--truth-dir", default="dist/truth")
     parser.add_argument("--platforms-dir", default="platforms")
     parser.add_argument("--emulators-dir", default="emulators")
     args = parser.parse_args()
@@ -979,6 +1063,10 @@ def main() -> int:
                 if args.offline
                 else check_platforms(platforms_dir, workdir, args.jobs)
             )
+        if "native" in areas:
+            findings.extend(
+                check_native(platforms_dir, Path(args.native_cache_dir), Path(args.truth_dir))
+            )
         if "targets" in areas:
             findings.extend(check_targets(platforms_dir, workdir, profiles, args.jobs, args.offline))
     if "coreinfo" in areas:

scripts/export_native.py

@@ -10,6 +10,7 @@ a platform ships keeps working.
 Usage:
     python scripts/export_native.py --all --fetch
     python scripts/export_native.py --platform recalbox --upstream-dir up/
+    python scripts/export_native.py --platform retrobat --refresh-cache
 """
 
 from __future__ import annotations
@@ -28,6 +29,7 @@ from exporter import discover_exporters
 from exporter.baseline import build_native_model
 
 DEFAULT_CACHE = ".cache/upstream-native"
+SOURCES_INDEX = ".sources.json"
 _USER_AGENT = "retrobios-exporter/1.0"
 _MAX_BYTES = 64 * 1024 * 1024
 
@@ -41,7 +43,29 @@ def _load_sources(index: Path | None) -> dict[str, str]:
         return {}
 
 
-def fetch(url: str, destination: Path, index: Path | None = None) -> bytes:
+def source_key(destination: Path, index: Path | None) -> str:
+    """How the index names a cached file: its path below the cache root.
+
+    A key that repeated the cache directory as it was typed changed with the
+    working directory, and a file recorded from one place looked unrecorded
+    from another.
+    """
+    if index is not None:
+        try:
+            return destination.resolve().relative_to(index.parent.resolve()).as_posix()
+        except ValueError:
+            pass
+    return str(destination)
+
+
+def load_sources(upstream_dir: Path) -> dict[str, str]:
+    """Cached file -> URL it was fetched from, for a whole cache directory."""
+    return _load_sources(upstream_dir / SOURCES_INDEX)
+
+
+def fetch(
+    url: str, destination: Path, index: Path | None = None, refresh: bool = False
+) -> bytes:
     """Download an original once, then read it from the cache.
 
     The cache path carries the file's own name and nothing of the revision it
@@ -50,10 +74,15 @@ def fetch(url: str, destination: Path, index: Path | None = None) -> bytes:
     URL that produced each cached file is recorded beside the cache, and a
     different URL refetches. A cache written before this index existed keeps
     being served: nothing recorded means nothing contradicted.
+
+    A branch URL names no revision, so the same URL serves new bytes after
+    the platform moves. refresh downloads again whatever the cache holds:
+    it is what a rescrape calls, so the original and its transcription
+    describe the same moment.
     """
     recorded = _load_sources(index)
-    key = str(destination)
-    if destination.exists() and recorded.get(key, url) == url:
+    key = source_key(destination, index)
+    if not refresh and destination.exists() and recorded.get(key, url) == url:
         return destination.read_bytes()
     request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
     with urllib.request.urlopen(request, timeout=60) as response:
@@ -94,14 +123,10 @@ def _raw_url(url: str) -> str:
     return url
 
 
-def collect_originals(
-    exporter: object,
-    systems: dict,
-    upstream_dir: Path,
-    allow_fetch: bool,
-    scraped: dict | None = None,
-) -> tuple[dict[str, str], list[str]]:
-    """Gather the platform's own files, from disk or from upstream."""
+def wanted_sources(
+    exporter: object, systems: dict, scraped: dict | None = None
+) -> dict[str, str]:
+    """Cache-relative name -> URL of every file the exporter patches."""
     wanted = dict(exporter.native_sources())
     components = getattr(exporter, "components", None)
     if callable(components):
@@ -113,21 +138,39 @@ def collect_originals(
     base = pinned_base(wanted, scraped)
     if base:
         wanted = {relative: base + relative for relative in wanted}
+    return wanted
 
+
+def collect_originals(
+    exporter: object,
+    systems: dict,
+    upstream_dir: Path,
+    allow_fetch: bool,
+    scraped: dict | None = None,
+    refresh: bool = False,
+) -> tuple[dict[str, str], list[str]]:
+    """Gather the platform's own files, from disk or from upstream.
+
+    With fetching on, an existing file still goes through fetch(): reading
+    it straight from disk skipped the recorded URL, so a file cached under
+    one pin kept being patched after the platform YAML moved to the next.
+    """
+    wanted = wanted_sources(exporter, systems, scraped)
     root = upstream_dir / exporter.platform_name()
+    index = upstream_dir / SOURCES_INDEX
     originals: dict[str, str] = {}
     missing: list[str] = []
     for relative, url in wanted.items():
         path = root / relative
         payload: bytes | None = None
-        if path.exists():
-            payload = path.read_bytes()
-        elif allow_fetch:
+        if allow_fetch:
             try:
-                payload = fetch(url, path, upstream_dir / ".sources.json")
+                payload = fetch(url, path, index, refresh)
             except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
                 missing.append(f"{relative}: {exc}")
                 continue
+        elif path.exists():
+            payload = path.read_bytes()
         else:
             missing.append(f"{relative}: absent and fetching is off")
             continue
@@ -141,6 +184,42 @@ def collect_originals(
     return originals, missing
 
 
+def load_inputs(
+    platform: str, truth_dir: Path, platforms_dir: str
+) -> tuple[dict | None, dict | None]:
+    """The truth and the scraped config of a platform, None where absent."""
+    truth_file = truth_dir / f"{platform}.yml"
+    truth: dict | None = None
+    if truth_file.exists():
+        with open(truth_file) as handle:
+            truth = yaml_load(handle) or {}
+    try:
+        scraped = load_platform_config(platform, platforms_dir)
+    except (FileNotFoundError, OSError):
+        scraped = None
+    return truth, scraped
+
+
+def refresh_cache(
+    platform: str,
+    exporter_class: type,
+    truth_dir: Path,
+    platforms_dir: str,
+    upstream_dir: Path,
+) -> tuple[bool, list[str]]:
+    """Download a platform's own files again. Returns (ok, messages)."""
+    truth, scraped = load_inputs(platform, truth_dir, platforms_dir)
+    systems, _ = build_native_model(truth or {}, scraped)
+    exporter = exporter_class()
+    wanted = wanted_sources(exporter, systems, scraped)
+    _, missing = collect_originals(
+        exporter, systems, upstream_dir, True, scraped, refresh=True
+    )
+    if missing:
+        return False, missing
+    return True, [f"{len(wanted)} file(s) fetched"]
+
+
 def export_platform(
     platform: str,
     exporter_class: type,
@@ -153,19 +232,11 @@ def export_platform(
     """Write one platform's corrected file. Returns (ok, messages)."""
     messages: list[str] = []
 
-    truth_file = truth_dir / f"{platform}.yml"
-    truth: dict = {}
-    if truth_file.exists():
-        with open(truth_file) as handle:
-            truth = yaml_load(handle) or {}
-    else:
+    truth, scraped = load_inputs(platform, truth_dir, platforms_dir)
+    if truth is None:
+        truth = {}
         messages.append(f"no truth for {platform}, only the platform's own data")
 
-    try:
-        scraped = load_platform_config(platform, platforms_dir)
-    except (FileNotFoundError, OSError):
-        scraped = None
-
     systems, report = build_native_model(truth, scraped)
     if not systems:
         return False, ["nothing to write: neither the platform nor the truth has data"]
@@ -253,6 +324,7 @@ def run(
     platforms_dir: str,
     upstream_dir: str,
     allow_fetch: bool,
+    refresh_only: bool = False,
 ) -> int:
     exporters = discover_exporters()
     failures = 0
@@ -265,15 +337,24 @@ def run(
             print(f"  SKIP {platform}: no exporter")
             continue
 
-        ok, messages = export_platform(
-            platform,
-            exporter_class,
-            Path(truth_dir),
-            Path(output_dir),
-            platforms_dir,
-            Path(upstream_dir),
-            allow_fetch,
-        )
+        if refresh_only:
+            ok, messages = refresh_cache(
+                platform,
+                exporter_class,
+                Path(truth_dir),
+                platforms_dir,
+                Path(upstream_dir),
+            )
+        else:
+            ok, messages = export_platform(
+                platform,
+                exporter_class,
+                Path(truth_dir),
+                Path(output_dir),
+                platforms_dir,
+                Path(upstream_dir),
+                allow_fetch,
+            )
         label = "OK  " if ok else "FAIL"
         print(f"  {label} {platform}")
         for message in messages:
@@ -306,7 +387,13 @@ def main() -> None:
     parser.add_argument(
         "--fetch",
         action="store_true",
-        help="download a platform's file when it is not in the cache",
+        help="download a platform's file when it is not in the cache, "
+        "or when the cache came from another URL",
+    )
+    parser.add_argument(
+        "--refresh-cache",
+        action="store_true",
+        help="download every platform file again and write no export",
     )
     args = parser.parse_args()
 
@@ -326,6 +413,7 @@ def main() -> None:
             args.platforms_dir,
             args.upstream_dir,
             args.fetch,
+            args.refresh_cache,
         )
     )
 

scripts/refresh_stale.py

@@ -10,6 +10,10 @@ committed.
 Mapping (stale -> command):
     platforms/<name>        python -m scripts.scraper.<module>_scraper
                             -o platforms/<name>.yml
+                            then the native refresh below: the original
+                            the export patches must be the one just scraped
+    native/<name>           python scripts/export_native.py --platform <name>
+                            --refresh-cache
     targets/<name>          python -m scripts.scraper.targets.<module>
                             -o platforms/targets/<name>.yml
     data/<key>              python scripts/refresh_data_dirs.py --key <key>
@@ -47,7 +51,7 @@ LOG_DIR = REPO_ROOT / "tmp" / "refresh_stale"
 CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
 PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
 
-AUTO_AREAS = ("platforms", "targets", "data", "catalogs")
+AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
 MANUAL_AREAS = ("coreinfo", "ci", "profiles")
 
 JOB_TIMEOUT = 1800  # 30 minutes per refresher; scrapers rarely exceed 10
@@ -61,6 +65,8 @@ class Job:
     subject: str
     command: list[str]
     log_path: Path
+    # Commands run after `command`, in order, only while each one succeeds.
+    then: tuple[tuple[str, ...], ...] = ()
 
 
 @dataclass(frozen=True)
@@ -122,6 +128,15 @@ def plan_jobs(
     seen: set[tuple[str, str]] = set()
     surfaced: list[dict] = []
     ignored: list[dict] = []
+    # A platform about to be rescraped refreshes its cached original itself,
+    # after the scrape: a separate native job would race the scraper.
+    rescraped = {
+        str(f.get("subject") or "")
+        for f in findings
+        if f.get("status") == "STALE"
+        and f.get("area") == "platforms"
+        and _command_for("platforms", str(f.get("subject") or ""), registry)
+    }
 
     for finding in findings:
         status = finding.get("status")
@@ -135,6 +150,10 @@ def plan_jobs(
             ignored.append(finding)
             continue
 
+        if area == "native" and subject in rescraped:
+            ignored.append(finding)
+            continue
+
         command = _command_for(area, subject, registry)
         if command is None:
             surfaced.append(finding)
@@ -148,15 +167,33 @@ def plan_jobs(
         seen.add(key)
 
         log = LOG_DIR / f"{area}__{_slug(subject)}.log"
-        jobs.append(Job(area=area, subject=subject, command=command, log_path=log))
+        then: tuple[tuple[str, ...], ...] = ()
+        if area == "platforms":
+            then = (tuple(_native_refresh(subject)),)
+        jobs.append(
+            Job(area=area, subject=subject, command=command, log_path=log, then=then)
+        )
 
     return jobs, surfaced, ignored
 
 
+def _native_refresh(platform: str) -> list[str]:
+    return [
+        sys.executable,
+        "scripts/export_native.py",
+        "--platform",
+        platform,
+        "--refresh-cache",
+    ]
+
+
 def _command_for(
     area: str, subject: str, registry: dict[str, dict]
 ) -> list[str] | None:
     """Map a stale finding to its refresh command, or None if manual."""
+    if area == "native":
+        return _native_refresh(subject)
+
     if area == "platforms":
         entry = registry.get(subject) or {}
         scraper = entry.get("scraper")
@@ -250,23 +287,27 @@ def run_job(job: Job, env: dict[str, str]) -> JobResult:
     """Execute one refresh command and collect its output."""
     start = time.monotonic()
     job.log_path.parent.mkdir(parents=True, exist_ok=True)
+    returncode = 0
     with job.log_path.open("w", encoding="utf-8") as log:
-        log.write(f"$ {' '.join(job.command)}\n")
-        log.flush()
-        try:
-            proc = subprocess.run(
-                job.command,
-                cwd=REPO_ROOT,
-                stdout=log,
-                stderr=subprocess.STDOUT,
-                env=env,
-                timeout=JOB_TIMEOUT,
-                check=False,
-            )
-            returncode = proc.returncode
-        except subprocess.TimeoutExpired:
-            log.write(f"\nTIMEOUT after {JOB_TIMEOUT}s\n")
-            returncode = 124
+        for command in (job.command, *job.then):
+            log.write(f"$ {' '.join(command)}\n")
+            log.flush()
+            try:
+                proc = subprocess.run(
+                    list(command),
+                    cwd=REPO_ROOT,
+                    stdout=log,
+                    stderr=subprocess.STDOUT,
+                    env=env,
+                    timeout=JOB_TIMEOUT,
+                    check=False,
+                )
+                returncode = proc.returncode
+            except subprocess.TimeoutExpired:
+                log.write(f"\nTIMEOUT after {JOB_TIMEOUT}s\n")
+                returncode = 124
+            if returncode != 0:
+                break
     duration = time.monotonic() - start
     tail = _log_tail(job.log_path)
     return JobResult(job=job, returncode=returncode, duration=duration, tail=tail)
@@ -388,6 +429,8 @@ def main() -> int:
               f"{len(ignored)} already clean/skipped")
         for job in sorted(jobs, key=lambda j: (j.area, j.subject)):
             print(f"  {job.area:10} {job.subject:28}  {' '.join(job.command)}")
+            for command in job.then:
+                print(f"  {'':10} {'':28}  then {' '.join(command)}")
         if surfaced:
             print("\n[manual review needed]")
             for f in surfaced: