chore: merge round 2 fixes (ops)

scripts/artifacts.py

@@ -217,6 +217,33 @@ def hold_artifact_lock(directory: str) -> Iterator[None]:
             else:
                 os.environ[HELD_LOCK_ENV] = previous
 
+
+# What a pipeline run writes at fixed paths, whatever its --output-dir.
+REPOSITORY_OUTPUTS = ("database.json", "install/", "README.md", "CONTRIBUTING.md", "docs/")
+
+
+@contextlib.contextmanager
+def hold_repository_outputs(root: str = ".") -> Iterator[None]:
+    """Hold the outputs a pipeline writes at fixed paths, for the whole run.
+
+    The output-directory lock let a run on tmp/packtest start beside a run
+    on dist/: both rewrote database.json, and the second to reach the
+    manifest step found install/ held after minutes of work, or, when the
+    steps did not overlap, rewrote the committed manifests from its own
+    state. The lock file is the repository root's .lock.
+    """
+    stack = contextlib.ExitStack()
+    try:
+        stack.enter_context(artifact_lock(root))
+    except ArtifactLockBusy as exc:
+        raise ArtifactLockBusy(
+            f"another pipeline run is writing {', '.join(REPOSITORY_OUTPUTS)} "
+            f"(lock: {os.path.join(root, '.lock')}). Wait for it to finish."
+        ) from exc
+    with stack:
+        yield
+
+
 def _build_timestamp(db: dict | None = None) -> str:
     """Timestamp for generated artifacts.
 

scripts/auto_fetch.py

@@ -101,6 +101,8 @@ def find_missing(config: dict, db: dict) -> list[dict]:
                 "system": sys_id,
                 "sha1": file_entry.get("sha1"),
                 "md5": file_entry.get("md5"),
+                "sha256": file_entry.get("sha256"),
+                "crc32": file_entry.get("crc32"),
                 "size": file_entry.get("size"),
                 "destination": file_entry.get("destination", name),
             })
@@ -112,7 +114,9 @@ def find_missing(config: dict, db: dict) -> list[dict]:
             "system": u.get("system") or "",
             "sha1": u.get("sha1"),
             "md5": u.get("md5"),
-            "size": None,
+            "sha256": u.get("sha256"),
+            "crc32": u.get("crc32"),
+            "size": u.get("size"),
             "destination": u.get("path") or u["name"],
             "emulator": u.get("emulator", ""),
         })
@@ -131,15 +135,49 @@ def _accepted(entry: dict, field: str) -> set[str]:
     return {str(v).strip().lower() for v in values if str(v).strip()}
 
 
+def _crc32s(entry: dict) -> set[str]:
+    return {value.removeprefix("0x").zfill(8) for value in _accepted(entry, "crc32")}
+
+
+def _size(entry: dict) -> int | None:
+    try:
+        return int(entry["size"]) if entry.get("size") is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+CRYPTO_HASHES = (
+    ("sha256", hashlib.sha256),
+    ("sha1", hashlib.sha1),
+    ("md5", hashlib.md5),
+)
+
+
+def verifiable(entry: dict) -> bool:
+    """Whether a network download can be proved to be this file.
+
+    Only a cryptographic hash proves it. A crc32 and a size can be met by
+    crafted bytes, so they identify a file the database already holds
+    (step 1) and never accept one a remote source serves.
+    """
+    return any(_accepted(entry, field) for field, _digest in CRYPTO_HASHES)
+
+
 def verify_content(data: bytes, expected: dict) -> bool:
-    """Verify downloaded content matches one of the expected hashes."""
-    sha1s = _accepted(expected, "sha1")
-    if sha1s:
-        return hashlib.sha1(data).hexdigest() in sha1s
-    md5s = _accepted(expected, "md5")
-    if md5s:
-        return hashlib.md5(data).hexdigest() in md5s
-    return False
+    """Whether downloaded bytes match every cryptographic hash declared.
+
+    Each declared field must hold the bytes' digest among its accepted
+    values. An entry declaring none is refused.
+    """
+    declared = [
+        (accepted, digest)
+        for field, digest in CRYPTO_HASHES
+        for accepted in [_accepted(expected, field)]
+        if accepted
+    ]
+    return bool(declared) and all(
+        digest(data).hexdigest() in accepted for accepted, digest in declared
+    )
 
 
 def _read_limited(
@@ -158,11 +196,20 @@ def step1_crossref_db(entry: dict, db: dict) -> str | None:
     for sha1 in sorted(_accepted(entry, "sha1")):
         if sha1 in files:
             return files[sha1]["path"]
-    by_md5 = db.get("indexes", {}).get("by_md5", {})
-    for md5 in sorted(_accepted(entry, "md5")):
-        sha1_match = by_md5.get(md5)
-        if sha1_match and sha1_match in files:
-            return files[sha1_match]["path"]
+    indexes = db.get("indexes", {})
+    for field, index in (("sha256", "by_sha256"), ("md5", "by_md5")):
+        by_hash = indexes.get(index, {})
+        for value in sorted(_accepted(entry, field)):
+            sha1_match = by_hash.get(value)
+            if sha1_match and sha1_match in files:
+                return files[sha1_match]["path"]
+    size = _size(entry)
+    if size is not None:
+        by_crc32 = indexes.get("by_crc32", {})
+        for crc in sorted(_crc32s(entry)):
+            sha1_match = by_crc32.get(crc)
+            if sha1_match in files and files[sha1_match].get("size") == size:
+                return files[sha1_match]["path"]
     return None
 
 
@@ -352,7 +399,7 @@ def fetch_missing(
     dry_run: bool = False,
 ) -> dict:
     """Run the 5-step auto-fetch pipeline for missing files."""
-    stats = {"found": 0, "not_found": 0, "errors": 0}
+    stats = {"found": 0, "not_found": 0, "errors": 0, "unverifiable": 0}
     still_missing = []
 
     for entry in missing:
@@ -366,6 +413,14 @@ def fetch_missing(
             stats["found"] += 1
             continue
 
+        if not verifiable(entry):
+            # Searched, a file of that name could only ever be refused.
+            print("    [-] Cannot be verified from the network: declares no "
+                  "cryptographic hash. Not searched, nothing written")
+            still_missing.append(entry)
+            stats["unverifiable"] += 1
+            continue
+
         if dry_run:
             print("    [DRY RUN] Would search branches, repos, archive.org")
             still_missing.append(entry)
@@ -511,7 +566,8 @@ def main() -> None:
         stats = result["stats"]
         print(
             f"\n  Results: {stats['found']} found, {stats['not_found']} not found, "
-            f"{stats['errors']} not fully searched"
+            f"{stats['errors']} not fully searched, "
+            f"{stats['unverifiable']} without a cryptographic hash"
         )
 
     if args.create_issues and all_still_missing:

scripts/check_freshness.py

@@ -518,7 +518,10 @@ def check_native(
                 wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
             )
             if reason:
-                reason += ", export_native.py --refresh-cache fetches it"
+                reason += (
+                    f", export_native.py --platform {name} --refresh-cache"
+                    " fetches it, then the export is written again"
+                )
             else:
                 reason = published_export_state(
                     name, exporter_class, truth_dir, platforms_dir, cache_dir, published
@@ -1299,7 +1302,15 @@ def summarize_profile_sync(report: object) -> list[Finding]:
                                     remote=announced, detail=detail))
             continue
         if row.get("skipped"):
-            findings.append(Finding("profiles", name, UNKNOWN, detail=str(row["skipped"])))
+            # A skip profile_sync counts as a defect (no repository answers,
+            # no resolvable pin) is unknown. One that is the profile's nature
+            # (an upstream declared gone, a host no forge API serves, no
+            # source_ref) is never checkable and does not hold the exit code.
+            from profile_sync import ProfileReport
+
+            reason = str(row["skipped"])
+            status = UNKNOWN if reason.startswith(ProfileReport.UNCHECKABLE) else SKIPPED
+            findings.append(Finding("profiles", name, status, detail=reason))
             continue
         review = row.get("needs_review") or 0
         pin, head = str(row.get("pin") or ""), str(row.get("head") or "")
@@ -1360,6 +1371,13 @@ def render(findings: list[Finding]) -> str:
     counts = {status: sum(1 for f in findings if f.status == status) for status in STATUSES}
     summary = ", ".join(f"{n} {status.lower()}" for status, n in counts.items() if n)
     lines.append(f"\nFRESHNESS: {summary}")
+    if counts[STALE]:
+        lines.append(
+            "refresh_stale.py runs the refresher behind every stale row "
+            "(--dry-run lists them first)"
+        )
+    if counts[UNKNOWN]:
+        lines.append("an unknown row is an upstream that could not be read: not up to date")
     return "\n".join(lines)
 
 
@@ -1420,7 +1438,7 @@ def main() -> int:
     areas = tuple(a.strip() for a in args.only.split(",")) if args.only else AREAS
     unknown = sorted(set(areas) - set(AREAS))
     if unknown:
-        parser.error(f"unknown area: {', '.join(unknown)}")
+        parser.error(unknown_areas_message(unknown))
 
     platforms_dir = Path(args.platforms_dir)
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
@@ -1455,7 +1473,28 @@ def main() -> int:
         print(json.dumps([asdict(f) for f in findings], indent=2))
     else:
         print(render(findings))
-    return 1 if any(f.status in (STALE, ERROR) for f in findings) else 0
+    return exit_code(findings)
+
+
+def exit_code(findings: list[Finding]) -> int:
+    """Zero only when every upstream asked answered and nothing is stale.
+
+    An unknown row is an upstream that could not be read: a run without
+    network gave 28 of them and exited 0, the answer of a fresh collection.
+    """
+    return 1 if any(f.status in (STALE, ERROR, UNKNOWN) for f in findings) else 0
+
+
+def unknown_areas_message(unknown: list[str]) -> str:
+    """The refusal of --only, naming the areas it accepts."""
+    import difflib
+
+    hints = [
+        f"{name} (did you mean {close[0]}?)" if close else name
+        for name in unknown
+        for close in [difflib.get_close_matches(name, AREAS, n=1)]
+    ]
+    return f"unknown area: {', '.join(hints)}. Areas: {', '.join(AREAS)}"
 
 
 if __name__ == "__main__":

scripts/common.py

@@ -2810,6 +2810,7 @@ from artifacts import (  # noqa: E402,F401
     ArtifactLockBusy,
     artifact_lock,
     hold_artifact_lock,
+    hold_repository_outputs,
     _TIMESTAMP_PATTERNS,
     _strip_timestamps,
 )

scripts/cross_reference.py

@@ -959,6 +959,15 @@ def main() -> None:
     if args.platform:
         require_platform(args.platform, args.platforms_dir)
 
+    import contextlib
+
+    from refresh_data_dirs import hold_data_dirs
+
+    # Files resolve against data/: the trees stay put until the process
+    # ends, which releases the hold.
+    held = contextlib.ExitStack()
+    hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
+
     profiles = load_emulator_profiles(args.emulators_dir)
     emulator = args.emulator
     if emulator:

scripts/export_native.py

@@ -899,6 +899,17 @@ def main() -> None:
         require_platform(args.platform, args.platforms_dir)
         platforms = [args.platform]
 
+    if not args.refresh_cache:
+        import contextlib
+
+        from refresh_data_dirs import hold_data_dirs
+
+        # Removal candidates are proved against data/: a refresh swapping a
+        # tree mid-walk listed entries only that tree holds as removable.
+        # The process exit releases the hold.
+        held = contextlib.ExitStack()
+        hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
+
     sys.exit(
         run(
             platforms,

scripts/pipeline.py

@@ -7,7 +7,7 @@ Steps, numbered as the run prints them:
   1b   provenance_report.py         dump-catalog coverage, derivable.json
   1c   romset_recipes.py            archive identification, reconstruction targets
   2    refresh_data_dirs.py         Dolphin Sys, PPSSPP assets and the other trees
-  2a   mame_hash_scraper            MAME BIOS hashes from the driver source
+  2a   mame_hash_scraper            MAME BIOS hashes, and device sets on a new release
   2a2  fbneo_hash_scraper           FBNeo BIOS hashes
   2b   check_buildbot_system.py     buildbot freshness (--check-buildbot)
   2b2  check_release_assets.py      large-files release vs gitignored files
@@ -68,6 +68,7 @@ from common import (
     artifact_lock,
     github_token,
     hold_artifact_lock,
+    hold_repository_outputs,
     load_data_dir_registry,
 )
 from refresh_data_dirs import hold_data_dirs
@@ -700,13 +701,20 @@ def main() -> None:
     # hold lasts until this process exits: taken step by step, it let another
     # run purge the packs between the build and the integrity check, which
     # then reported every platform SKIP and passed.
+    # database.json, install/, README.md and docs/ sit at fixed paths, so
+    # every run holds them too: a run on a scratch --output-dir otherwise
+    # rewrote them beside the run on dist/.
     held = contextlib.ExitStack()
-    if not args.skip_packs:
-        try:
+    try:
+        held.enter_context(
+            hold_repository_outputs(str(Path(__file__).resolve().parent.parent))
+        )
+        if not args.skip_packs:
             held.enter_context(hold_artifact_lock(args.output_dir))
-        except ArtifactLockBusy as exc:
-            print(f"ERROR: {exc}", file=sys.stderr)
-            sys.exit(1)
+    except ArtifactLockBusy as exc:
+        held.close()
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)
 
     all_ok = True
     total_start = time.monotonic()

scripts/provenance_report.py

@@ -10,6 +10,10 @@ Usage:
     python scripts/provenance_report.py
     python scripts/provenance_report.py --missing
     python scripts/provenance_report.py --json
+
+A missing dump that is a file an emulator profile reads and the collection
+lacks is named with that profile: by hash when the profile declares one, by
+name and size when it declares a size and no hash.
 """
 
 from __future__ import annotations
@@ -26,10 +30,117 @@ sys.path.insert(0, os.path.dirname(__file__))
 from common import (
     DEFAULT_PROVENANCE_DIR,
     load_database,
+    load_emulator_profiles,
     load_provenance_snapshots,
     write_if_changed,
 )
 
+HASH_FIELDS = ("sha1", "md5", "sha256")
+
+
+def _values(entry: dict, field: str) -> set[str]:
+    """Every value a profile or catalog entry gives for a hash field."""
+    value = entry.get(field)
+    values = value if isinstance(value, list) else str(value or "").split(",")
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _crc_key(value: str) -> str:
+    return value.removeprefix("0x").zfill(8)
+
+
+def _fits(size: int | None, entry: dict) -> bool:
+    """Whether a size meets what a profile entry states about it."""
+    if size is None:
+        return False
+    declared = entry.get("size")
+    if declared is not None:
+        accepted = declared if isinstance(declared, list) else [declared]
+        return size in {int(value) for value in accepted}
+    low, high = entry.get("min_size"), entry.get("max_size")
+    return (low is None or size >= int(low)) and (high is None or size <= int(high))
+
+
+def profile_needs(profiles: dict, db: dict) -> dict:
+    """The files emulator profiles read that the collection does not hold.
+
+    Keyed by every hash a profile accepts, and, for an entry that declares
+    a size and no hash, by its names casefolded. An entry with neither
+    cannot be told apart from any same-named dump and is left out.
+    """
+    files = db.get("files", {})
+    indexes = db.get("indexes", {})
+    held_hashes = {
+        "sha1": set(files),
+        "md5": set(indexes.get("by_md5", {})),
+        "sha256": set(indexes.get("by_sha256", {})),
+    }
+    held_crc = {
+        (_crc_key(str(f.get("crc32", "")).lower()), f.get("size"))
+        for f in files.values() if f.get("crc32")
+    }
+    sizes_by_name: dict[str, set[int]] = {}
+    for record in files.values():
+        sizes_by_name.setdefault(str(record.get("name", "")).casefold(), set()).add(
+            record.get("size")
+        )
+
+    by_hash: dict[tuple, list[dict]] = {}
+    by_name: dict[str, list[tuple[dict, dict]]] = {}
+    for emulator, profile in sorted(profiles.items()):
+        for entry in profile.get("files", []) or []:
+            name = str(entry.get("name", ""))
+            if not name or name.endswith("/"):
+                continue
+            need = {"emulator": emulator, "name": name,
+                    "required": bool(entry.get("required"))}
+            hashes = {(field, v) for field in HASH_FIELDS for v in _values(entry, field)}
+            crcs = {_crc_key(v) for v in _values(entry, "crc32")}
+            declared = entry.get("size")
+            sizes = [] if declared is None else (
+                declared if isinstance(declared, list) else [declared]
+            )
+            crc_keys = {(crc, int(size)) for crc in crcs for size in sizes}
+            if hashes or crc_keys:
+                if any(v in held_hashes[field] for field, v in hashes) or (
+                    crc_keys & held_crc
+                ):
+                    continue
+                for key in hashes | {("crc32", crc, size) for crc, size in crc_keys}:
+                    by_hash.setdefault(key, []).append(need)
+                continue
+            if all(entry.get(k) is None for k in ("size", "min_size", "max_size")):
+                continue
+            names = [name, *(entry.get("aliases") or [])]
+            if any(
+                _fits(held, entry)
+                for alias in names
+                for held in sizes_by_name.get(str(alias).casefold(), ())
+            ):
+                continue
+            for alias in names:
+                by_name.setdefault(str(alias).casefold(), []).append((need, entry))
+    return {"by_hash": by_hash, "by_name": by_name}
+
+
+def needs_filled(entry: dict, needs: dict) -> list[dict]:
+    """The profile needs a catalog dump answers, required ones first."""
+    found: dict[tuple[str, str], dict] = {}
+    keys = [(field, v) for field in HASH_FIELDS for v in _values(entry, field)]
+    if entry.get("size") is not None:
+        keys += [("crc32", _crc_key(v), entry["size"]) for v in _values(entry, "crc32")]
+    for key in keys:
+        for need in needs["by_hash"].get(key, []):
+            found.setdefault((need["emulator"], need["name"]), {**need, "by": key[0]})
+    size = entry.get("size")
+    name = str(entry.get("name", "")).casefold()
+    for need, declared in needs["by_name"].get(name, []):
+        if _fits(size, declared):
+            found.setdefault(
+                (need["emulator"], need["name"]), {**need, "by": "name and size"}
+            )
+    return sorted(found.values(), key=lambda n: (not n["required"], n["emulator"], n["name"]))
+
 
 def archive_members(db: dict) -> dict[tuple[str, int], list[tuple[str, str]]]:
     """(crc32, size) of every member of the collection's ZIPs -> (zip, member).
@@ -153,6 +264,7 @@ def build_report(
     snapshots: dict,
     members: dict | None = None,
     romsets: dict | None = None,
+    needs: dict | None = None,
 ) -> dict:
     """Compare each snapshot against the collection.
 
@@ -213,6 +325,9 @@ def build_report(
             holders = romsets_holding(entry, romsets)
             if holders:
                 entry["romsets"] = holders
+            filled = needs_filled(entry, needs) if needs else []
+            if filled:
+                entry["needed_by"] = filled
         out_of_scope = len({
             dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats
         })
@@ -245,6 +360,10 @@ def main() -> int:
         "byte swaps, concatenations). Reads the collection, about 35 s",
     )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
+    parser.add_argument(
+        "--emulators-dir", default="emulators",
+        help="profiles whose unheld files a missing dump answers (default: %(default)s)",
+    )
     parser.add_argument(
         "--write-derivable",
         metavar="PATH",
@@ -261,7 +380,8 @@ def main() -> int:
         print(f"No provenance snapshots in {args.provenance_dir}/")
         return 0
 
-    report = build_report(db, snapshots, archive_members(db))
+    needs = profile_needs(load_emulator_profiles(args.emulators_dir), db)
+    report = build_report(db, snapshots, archive_members(db), needs=needs)
     derived: dict[str, dict] = {}
     if args.derive:
         from derivations import derivable  # noqa: PLC0415
@@ -299,7 +419,8 @@ def main() -> int:
                 f"does not cover, not counted as targets"
             )
         if args.missing:
-            for entry in data["missing"]:
+            # A dump an emulator reads comes first: it fills a hole in a pack.
+            for entry in sorted(data["missing"], key=lambda e: not e.get("needed_by")):
                 label = entry.get("description") or entry["name"]
                 source = derived.get(entry.get("sha1", ""))
                 if source:
@@ -307,6 +428,10 @@ def main() -> int:
                           f"of {' + '.join(source['from'])}")
                 else:
                     print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
+                for need in entry.get("needed_by", []):
+                    print(f"      needed by {need['emulator']} as {need['name']} "
+                          f"({'required' if need['required'] else 'optional'}, "
+                          f"matched by {need['by']})")
                 for hit in entry.get("romsets", []):
                     versions = hit["dats"][0] if len(hit["dats"]) == 1 else (
                         f"{hit['dats'][0]} to {hit['dats'][-1]}"
@@ -320,6 +445,13 @@ def main() -> int:
         if args.derive:
             line += f", {len(derived)} of them derivable from held dumps"
         print(line)
+        needed = {
+            dump_identity(e)
+            for data in report.values() for e in data["missing"] if e.get("needed_by")
+        }
+        if needed:
+            print(f"  {len(needed)} of them a profile reads and the collection lacks"
+                  + ("" if args.missing else " (--missing names them)"))
     return 0
 
 

scripts/refresh_data_dirs.py

@@ -99,11 +99,13 @@ def hold_data_dirs(stack: contextlib.ExitStack, registry: dict) -> None:
     later. A refresh swapping the tree in between shipped bytes nobody
     verified, or died on a file the new tree no longer has. A refresh waits
     for the holders. Never called before a refresh in the same process:
-    the exclusive lock would wait on this one.
+    the exclusive lock would wait on this one. A cache not yet on disk is
+    held too: the lock sits beside the tree, and skipped, a refresh could
+    install the tree after verify ran and before the packs read it.
     """
     for entry in registry.values():
         cache = entry.get("local_cache", "")
-        if cache and os.path.isdir(cache):
+        if cache:
             stack.enter_context(cache_lock(cache, shared=True))
 
 

scripts/refresh_stale.py

@@ -11,7 +11,10 @@ Mapping (stale -> command):
     platforms/<name>        python -m scripts.scraper.<module>_scraper
                             -o platforms/<name>.yml
                             then the native refresh below: the original
-                            the export patches must be the one just scraped
+                            the export patches must be the one just scraped,
+                            then python scripts/verify.py --platform <name>,
+                            whose summary the report prints: a rescrape can
+                            add systems whose files are missing
     native/<name>           python scripts/generate_truth.py --platform <name>, then
                             python scripts/export_native.py --platform <name>
                             --refresh-cache, then the export itself, which
@@ -25,6 +28,10 @@ Mapping (stale -> command):
     catalogs/fbneo recipes  python -m scripts.scraper.romset_dat_importer
                             --source fbneo --fetch
 
+A platform that inherits from a rescraped one (retropie and lakka from
+retroarch) reads the parent's file: its jobs start once the parent's chain
+has ended, and its export is rewritten even when no row named it.
+
 Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
 (the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
 CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
@@ -45,7 +52,7 @@ import re
 import subprocess
 import sys
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
@@ -72,6 +79,8 @@ class Job:
     log_path: Path
     # Commands run after `command`, in order, only while each one succeeds.
     then: tuple[tuple[str, ...], ...] = ()
+    # Platforms whose rescrape job must end before this one starts.
+    after: frozenset[str] = frozenset()
 
 
 @dataclass(frozen=True)
@@ -94,6 +103,30 @@ def _load_platform_registry() -> dict[str, dict]:
         return (yaml.safe_load(fh) or {}).get("platforms") or {}
 
 
+def _platform_parents(registry: dict[str, dict]) -> dict[str, str]:
+    """Platform to the platform its YAML inherits from."""
+    import yaml
+
+    parents: dict[str, str] = {}
+    for name in registry:
+        path = REPO_ROOT / "platforms" / f"{name}.yml"
+        if not path.is_file():
+            continue
+        with path.open(encoding="utf-8") as fh:
+            parent = (yaml.safe_load(fh) or {}).get("inherits")
+        if parent:
+            parents[name] = str(parent)
+    return parents
+
+
+def _ancestors(platform: str, parents: dict[str, str]) -> set[str]:
+    found: set[str] = set()
+    while platform in parents and parents[platform] not in found:
+        platform = parents[platform]
+        found.add(platform)
+    return found
+
+
 @lru_cache(maxsize=1)
 def _data_dir_keys() -> frozenset[str]:
     """The entries refresh_data_dirs.py --key accepts."""
@@ -129,15 +162,21 @@ def _slug(subject: str) -> str:
 
 
 def plan_jobs(
-    findings: list[dict], registry: dict[str, dict]
+    findings: list[dict],
+    registry: dict[str, dict],
+    parents: dict[str, str] | None = None,
 ) -> tuple[list[Job], list[dict], list[dict]]:
     """Split findings into (refreshable jobs, surfaced-only, ignored).
 
     Surfaced-only entries carry stale state that only a human can resolve
     (new .info file, manual DAT download, PyPI bump). Ignored entries are
     anything not STALE (OK, SKIPPED, UNKNOWN, ERROR). ERROR is also surfaced
-    so the final report mentions it.
+    so the final report mentions it. ``parents`` maps a platform to the one
+    it inherits from: a child's jobs wait for its rescraped ancestors, and a
+    child no row names gets its export rewritten after them.
     """
+    if parents is None:
+        parents = _platform_parents(registry)
     jobs: list[Job] = []
     seen: set[tuple[str, str]] = set()
     surfaced: list[dict] = []
@@ -189,16 +228,92 @@ def plan_jobs(
                 tuple(_truth_refresh(subject)),
                 tuple(_native_refresh(subject)),
                 tuple(_export_refresh(subject)),
+                tuple(_verify(subject)),
             )
         elif area == "native":
             then = (tuple(_native_refresh(subject)), tuple(_export_refresh(subject)))
+        after: frozenset[str] = frozenset()
+        if area in ("platforms", "native"):
+            after = frozenset(_ancestors(subject, parents) & rescraped)
         jobs.append(
-            Job(area=area, subject=subject, command=command, log_path=log, then=then)
+            Job(area=area, subject=subject, command=command, log_path=log,
+                then=then, after=after)
         )
 
+    # The export of a child is built from its parent's file: rewritten once
+    # the parent is, or the next check reports it stale.
+    covered = {j.subject for j in jobs if j.area in ("platforms", "native")}
+    for child in sorted(parents):
+        after = frozenset(_ancestors(child, parents) & rescraped)
+        if not after or child in covered:
+            continue
+        jobs.append(Job(
+            area="native",
+            subject=child,
+            command=_truth_refresh(child),
+            log_path=LOG_DIR / f"native__{_slug(child)}.log",
+            then=(tuple(_native_refresh(child)), tuple(_export_refresh(child))),
+            after=after,
+        ))
+
     return jobs, surfaced, ignored
 
 
+def _verify(platform: str) -> list[str]:
+    """The check a rescraped platform file is read with before it is kept."""
+    return [sys.executable, "scripts/verify.py", "--platform", platform]
+
+
+VERIFY_MARK = "scripts/verify.py --platform"
+
+
+def verify_summary(path: Path) -> str:
+    """First line verify printed in a job log, empty when it did not run."""
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+    lines = text.splitlines()
+    for index, line in enumerate(lines):
+        if line.startswith("$ ") and VERIFY_MARK in line:
+            return next(
+                (rest.strip() for rest in lines[index + 1:]
+                 if rest.strip() and not rest.startswith("$ ")),
+                "",
+            )
+    return ""
+
+
+def run_jobs(jobs: list[Job], env: dict[str, str], workers: int) -> list[JobResult]:
+    """Run jobs in parallel, each once the rescrapes it waits for have ended.
+
+    A failed parent still releases its children: its file is unchanged, and
+    the child reads it as it is.
+    """
+    results: list[JobResult] = []
+    ended: set[str] = set()
+    waiting = list(jobs)
+    with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
+        running: dict = {}
+        while waiting or running:
+            ready = [job for job in waiting if job.after <= ended]
+            for job in ready:
+                waiting.remove(job)
+                running[pool.submit(run_job, job, env)] = job
+            if not running:
+                raise RuntimeError(
+                    "jobs wait on rescrapes that are not planned: "
+                    + ", ".join(sorted(j.subject for j in waiting))
+                )
+            done, _ = wait(running, return_when=FIRST_COMPLETED)
+            for future in done:
+                job = running.pop(future)
+                results.append(future.result())
+                if job.area == "platforms":
+                    ended.add(job.subject)
+    return results
+
+
 def _truth_refresh(platform: str) -> list[str]:
     return [sys.executable, "scripts/generate_truth.py", "--platform", platform]
 
@@ -380,6 +495,22 @@ def render(
             if r.returncode != 0 and r.tail:
                 lines.append(f"           -> {r.tail}")
 
+        rescraped = sorted(
+            (r for r in results if r.job.area == "platforms" and r.returncode == 0),
+            key=lambda r: r.job.subject,
+        )
+        if rescraped:
+            lines.append(
+                "\n[verify] a rescrape can add systems: read each report "
+                "before keeping platforms/<name>.yml"
+            )
+            for r in rescraped:
+                summary = verify_summary(r.job.log_path) or "(no verify output)"
+                lines.append(f"  {r.job.subject:12} {summary}")
+                lines.append(
+                    f"  {'':12} python scripts/verify.py --platform {r.job.subject}"
+                )
+
     if surfaced:
         lines.append("\n[manual review needed]")
         for f in surfaced:
@@ -439,7 +570,9 @@ def main() -> int:
         areas = tuple(a.strip() for a in args.only.split(",") if a.strip())
         unknown = sorted(set(areas) - set(all_areas))
         if unknown:
-            parser.error(f"unknown area: {', '.join(unknown)}")
+            parser.error(
+                f"unknown area: {', '.join(unknown)}. Areas: {', '.join(all_areas)}"
+            )
     else:
         areas = all_areas
 
@@ -457,6 +590,8 @@ def main() -> int:
               f"{len(ignored)} already clean/skipped")
         for job in sorted(jobs, key=lambda j: (j.area, j.subject)):
             print(f"  {job.area:10} {job.subject:28}  {' '.join(job.command)}")
+            if job.after:
+                print(f"  {'':10} {'':28}  after {', '.join(sorted(job.after))}")
             for command in job.then:
                 print(f"  {'':10} {'':28}  then {' '.join(command)}")
         if surfaced:
@@ -468,11 +603,7 @@ def main() -> int:
 
     LOG_DIR.mkdir(parents=True, exist_ok=True)
     env = _github_token_env()
-    results: list[JobResult] = []
-    with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool:
-        futures = [pool.submit(run_job, job, env) for job in jobs]
-        for future in as_completed(futures):
-            results.append(future.result())
+    results = run_jobs(jobs, env, args.jobs)
 
     if args.json:
         payload = {

scripts/release_notes.py

@@ -14,6 +14,7 @@ Usage:
 from __future__ import annotations
 
 import argparse
+import re
 import sys
 from pathlib import Path
 
@@ -37,18 +38,27 @@ def _display(pack: str) -> str:
 
 def _extract_folders(record: dict, platforms_dir: str) -> dict[str, str]:
     """Pack name to the folder the registry says it extracts into."""
+    registry = load_platform_registry(platforms_dir)
     folders: dict[str, str] = {}
-    for key, entry in sorted(load_platform_registry(platforms_dir).items()):
-        if not entry.get("extract_to"):
-            continue
-        needle = release_record._match_key(key)
-        for name in sorted(record.get("packs", {})):
-            # The name a pack starts with: RetroArch_Lakka is RetroArch's.
-            if needle and release_record._match_key(name).startswith(needle):
-                folders.setdefault(name, entry["extract_to"])
+    for name, pack in sorted(record.get("packs", {}).items()):
+        # A shared pack extracts where the platform it is named after does:
+        # RetroArch_Lakka is RetroArch's.
+        key = _match_key(name)
+        for platform in sorted(
+            pack.get("platforms", ()), key=lambda p: key.find(_match_key(p)) % (len(key) + 1)
+        ):
+            folder = registry.get(platform, {}).get("extract_to")
+            if folder:
+                folders[name] = folder
+                break
     return folders
 
 
+def _match_key(value: str) -> str:
+    """Letters and digits only: the platform is `misterfpga`, the pack MiSTer_FPGA."""
+    return re.sub(r"[^a-z0-9]", "", value.lower())
+
+
 def _links(tag: str, assets: list[str]) -> str:
     base = f"https://github.com/{REPO}/releases/download/{tag}"
     if len(assets) == 1:

scripts/release_record.py

@@ -18,39 +18,70 @@ from __future__ import annotations
 
 import argparse
 import json
-import re
 import sys
 import zipfile
 from pathlib import Path
 
 import split_pack
-from common import ArtifactLockBusy, artifact_lock, write_text_atomic
+from common import (
+    ArtifactLockBusy,
+    artifact_lock,
+    list_registered_platforms,
+    write_text_atomic,
+)
 
 RECORD = "release.json"
 
 
-def _match_key(value: str) -> str:
-    """Letters and digits only: the platform is `misterfpga`, the pack MiSTer_FPGA."""
-    return re.sub(r"[^a-z0-9]", "", value.lower())
-
-
 class UnfinishedSplitError(Exception):
     """A pack lies beside its own parts: a split is running or was killed."""
 
 
-def build_record(dist: Path, tag: str) -> dict:
+class NotAReleasePackError(Exception):
+    """A pack in the directory is not the full pack of a registered platform."""
+
+
+def release_owners(platforms_dir: str = "platforms") -> dict[str, list[str]]:
+    """Full pack file name to the platforms it serves, as the builder names it.
+
+    Rebuilt from the name builder, never matched by substring: matched by
+    substring, RetroArch_v1.22.2_AndroidArm64V8A_BIOS_Pack.zip sorts before
+    the full pack and was taken for the RetroArch pack.
+    """
+    import generate_pack
+
+    platforms = list_registered_platforms(platforms_dir, include_archived=True)
+    owners: dict[str, list[str]] = {}
+    for platform, names in generate_pack._expected_pack_names(
+        platforms, platforms_dir, None, None
+    ).items():
+        for name in names:
+            owners.setdefault(name, []).append(platform)
+    return {name: sorted(serves) for name, serves in owners.items()}
+
+
+def build_record(
+    dist: Path,
+    tag: str,
+    platforms_dir: str = "platforms",
+    owners: dict[str, list[str]] | None = None,
+) -> dict:
     """Read every pack of a directory, parts counted once across them.
 
     Read under the shared artifact lock, so a split that is reading its parts
     back cannot be recorded half done. A pack still beside its parts after
     that is a split that never finished: summed, it doubled the download size
-    and named an asset that is never published.
+    and named an asset that is never published. Each pack carries the
+    platforms it serves, so a reader finds a platform's pack by that list
+    after a later version or grouping has renamed the pack main would build.
     """
+    if owners is None:
+        owners = release_owners(platforms_dir)
     with artifact_lock(str(dist), exclusive=False):
-        return _read_record(Path(dist), tag)
+        return _read_record(Path(dist), tag, owners)
 
 
-def _read_record(dist: Path, tag: str) -> dict:
+def _read_record(dist: Path, tag: str, owners: dict[str, list[str]]) -> dict:
     assets: dict[str, list[Path]] = {}
     for path in sorted(dist.glob("*_BIOS_Pack*.zip")):
         assets.setdefault(split_pack.pack_of(path.name), []).append(path)
@@ -60,6 +91,12 @@ def _read_record(dist: Path, tag: str) -> dict:
             raise UnfinishedSplitError(
                 f"{whole[0].name} lies beside its parts: the split did not finish"
             )
+    foreign = sorted(name for name in assets if name not in owners)
+    if foreign:
+        raise NotAReleasePackError(
+            f"not the full pack of a registered platform: {', '.join(foreign)}"
+            " (a narrowed or older build left in the directory)"
+        )
 
     packs = {}
     for name, paths in sorted(assets.items()):
@@ -73,6 +110,7 @@ def _read_record(dist: Path, tag: str) -> dict:
             "extracted_size": sum(members.values()),
             "download_size": sum(path.stat().st_size for path in paths),
             "assets": [path.name for path in paths],
+            "platforms": owners[name],
         }
     return {"tag": tag, "packs": packs}
 
@@ -86,15 +124,18 @@ def load_record(path: str | Path = RECORD) -> dict:
         return {}
 
 
+def pack_name_for(platform: str, record: dict) -> str | None:
+    """Name of the release pack that serves a platform, if it has one."""
+    for name, pack in sorted(record.get("packs", {}).items()):
+        if platform in pack.get("platforms", ()):
+            return name
+    return None
+
+
 def pack_for(platform: str, record: dict) -> dict | None:
     """The pack of the release that serves a platform, if it has one."""
-    needle = _match_key(platform)
-    if not needle:
-        return None
-    for name, pack in sorted(record.get("packs", {}).items()):
-        if needle in _match_key(name):
-            return pack
-    return None
+    name = pack_name_for(platform, record)
+    return record["packs"][name] if name else None
 
 
 def manifest_mismatches(record: dict, install_dir: str | Path) -> list[str]:
@@ -103,17 +144,15 @@ def manifest_mismatches(record: dict, install_dir: str | Path) -> list[str]:
     for manifest_path in sorted(Path(install_dir).glob("*.json")):
         manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
         expected = manifest.get("pack_files")
-        if expected is None:
+        name = pack_name_for(manifest_path.stem, record)
+        if expected is None or name is None:
             continue
-        for name, pack in sorted(record.get("packs", {}).items()):
-            if _match_key(manifest_path.stem) not in _match_key(name):
-                continue
-            if pack["files"] != expected:
-                found.append(
-                    f"{name} holds {pack['files']} files, "
-                    f"install/{manifest_path.name} expects {expected}"
-                )
-            break
+        pack = record["packs"][name]
+        if pack["files"] != expected:
+            found.append(
+                f"{name} holds {pack['files']} files, "
+                f"install/{manifest_path.name} expects {expected}"
+            )
     return found
 
 
@@ -131,6 +170,10 @@ def main() -> int:
         "--install-dir", default="install", type=Path,
         help="manifests the archives are compared with (default: %(default)s)",
     )
+    parser.add_argument(
+        "--platforms-dir", default="platforms",
+        help="registry the pack names are rebuilt from (default: %(default)s)",
+    )
     parser.add_argument(
         "--output", default=RECORD, type=Path,
         help="record to write (default: %(default)s)",
@@ -138,8 +181,8 @@ def main() -> int:
     args = parser.parse_args()
 
     try:
-        record = build_record(args.dist, args.tag)
-    except (UnfinishedSplitError, ArtifactLockBusy) as exc:
+        record = build_record(args.dist, args.tag, args.platforms_dir)
+    except (UnfinishedSplitError, NotAReleasePackError, ArtifactLockBusy) as exc:
         print(f"Error: {exc}", file=sys.stderr)
         return 1
     if not record["packs"]:

scripts/scraper/mame_hash_scraper.py

@@ -334,6 +334,8 @@ def _run(args: argparse.Namespace) -> None:
         print("  no profiles with mamedev/mame upstream found")
         return
 
+    main_profile = _EMULATORS_DIR / "mame.yml"
+    pinned_before = _pinned_version(main_profile)
     for profile_path in profiles:
         is_main = profile_path.name == "mame.yml"
         diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame")
@@ -356,6 +358,34 @@ def _run(args: argparse.Namespace) -> None:
     print()
     if args.dry_run:
         print("(dry run, no files modified)")
+        return
+    pinned_after = _pinned_version(main_profile)
+    if pinned_after != pinned_before:
+        # The device sets describe the release mame.yml is pinned on: left
+        # behind, the profile names two releases at once.
+        print(f"mame.yml moved from {pinned_before} to {pinned_after}: "
+              "merging the device ROM sets of that release")
+        _merge_devices(argparse.Namespace(
+            listxml=fetch_release_listxml(pinned_after), dry_run=False,
+        ))
+
+
+def _pinned_version(profile_path: Path) -> str:
+    profile = yaml_load(profile_path.read_text(encoding="utf-8")) if profile_path.is_file() else {}
+    return str((profile or {}).get("core_version") or "")
+
+
+def fetch_release_listxml(version: str) -> Path:
+    """The -listxml MAME publishes for a release, in the DAT importer's cache.
+
+    0.289 is the tag mame0289 and the asset mame0289lx.zip.
+    """
+    from .romset_dat_importer import MAME_LISTXML_URL, _download
+
+    tag = "mame" + version.replace(".", "")
+    return _download(
+        MAME_LISTXML_URL.format(tag=tag), _ROOT / ".cache" / "dats" / f"{tag}lx.zip"
+    )
 
 
 def release_profile(

scripts/scraper/romset_dat_importer.py

@@ -11,6 +11,7 @@ in its repository, so both are fetched without a browser:
     python -m scripts.scraper.romset_dat_importer --source mame --fetch
     python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0289
     python -m scripts.scraper.romset_dat_importer --source fbneo --fetch
+    python -m scripts.scraper.romset_dat_importer --source fbneo --fetch v1.0.0.02
     python -m scripts.scraper.romset_dat_importer --source mame --pack local.dat
 
 Only sets the collection has a reason to know are kept, so the committed
@@ -33,6 +34,7 @@ import os
 import re
 import shutil
 import sys
+import urllib.parse
 import urllib.request
 import zipfile
 from collections.abc import Callable, Iterator
@@ -374,7 +376,7 @@ MAME_LISTXML_URL = (
     "https://github.com/mamedev/mame/releases/download/{tag}/{tag}lx.zip"
 )
 MAME_LATEST_API = "https://api.github.com/repos/mamedev/mame/releases/latest"
-FBNEO_DATS_API = "https://api.github.com/repos/libretro/FBNeo/contents/dats"
+FBNEO_API = "https://api.github.com/repos/libretro/FBNeo"
 
 
 def _api_json(url: str) -> object:
@@ -400,7 +402,8 @@ def latest_mame_tag() -> str:
 
 
 def fetch_tag(source: str, requested: str) -> str:
-    """The tag a ``--fetch`` names: bare, the newest MAME release."""
+    """The revision a ``--fetch`` names: bare, the newest MAME release or
+    the tip of FBNeo's default branch."""
     if source == "mame" and requested == "latest":
         tag = latest_mame_tag()
         print(f"  Derniere release MAME : {tag}")
@@ -408,6 +411,10 @@ def fetch_tag(source: str, requested: str) -> str:
     return requested
 
 
+class BlobMismatchError(ValueError):
+    """A downloaded DAT is not the blob the repository listing names."""
+
+
 def git_blob_sha(data: bytes) -> str:
     """The sha git gives a blob, so a cached file can be checked against a
     repository listing without any other state."""
@@ -452,7 +459,15 @@ def fetch_pack(
             {},
         )
 
-    listing = _api_json(FBNEO_DATS_API)
+    # The revision is resolved to a commit first and everything is read at
+    # that commit: a branch URL is served through a CDN cache, so the bytes
+    # could predate the listing that named their blob.
+    ref = "HEAD" if tag == "latest" else urllib.parse.quote(tag, safe="")
+    commit = _api_json(f"{FBNEO_API}/commits/{ref}")
+    sha = commit.get("sha") if isinstance(commit, dict) else None
+    if not sha:
+        raise BlobMismatchError(f"libretro/FBNeo has no revision {tag}")
+    listing = _api_json(f"{FBNEO_API}/contents/dats?ref={sha}")
     target = cache_dir / "fbneo-dats"
     target.mkdir(parents=True, exist_ok=True)
     blobs: dict[str, str] = {}
@@ -465,7 +480,17 @@ def fetch_pack(
         if destination.is_file() and git_blob_sha(destination.read_bytes()) != item["sha"]:
             destination.unlink()
         _download(item["download_url"], destination)
+        if git_blob_sha(destination.read_bytes()) != item["sha"]:
+            destination.unlink()
+            raise BlobMismatchError(
+                f"{item['name']} downloaded from {item['download_url']} is not "
+                f"blob {item['sha']} of libretro/FBNeo {sha}"
+            )
         blobs[item["name"]] = item["sha"]
+    # A DAT the revision does not carry is not read from an older fetch.
+    for stale in target.glob("*"):
+        if stale.is_file() and stale.name not in blobs:
+            stale.unlink()
     return target, blobs
 
 
@@ -509,8 +534,9 @@ def main() -> int:
         "--fetch",
         nargs="?",
         const="latest",
-        help="download upstream instead of using --pack "
-        "(a MAME tag such as mame0289, or bare for the newest release)",
+        help="download upstream instead of using --pack: a MAME tag such as "
+        "mame0289 or an FBNeo tag, branch or commit; bare, the newest MAME "
+        "release or the tip of FBNeo's default branch",
     )
     parser.add_argument(
         "--cache-dir", default=".cache/dats",
@@ -547,11 +573,24 @@ def main() -> int:
     args = parser.parse_args()
 
     blobs: dict[str, str] = {}
+    # FBNeo's DAT headers keep one version across commits, so a DAT read at
+    # a named revision is labelled with it: under the bare header name it
+    # merged into the tip's recipes. Its blobs describe that revision, not
+    # the upstream state the freshness check compares with, and are not
+    # recorded.
+    revision = ""
     if args.fetch:
-        pack, blobs = fetch_pack(
-            args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir)
-        )
+        try:
+            pack, blobs = fetch_pack(
+                args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir)
+            )
+        except BlobMismatchError as exc:
+            print(f"ERROR: {exc}", file=sys.stderr)
+            return 1
         print(f"  Recupere {pack}")
+        if args.source == "fbneo" and args.fetch != "latest":
+            revision = args.fetch
+            blobs = {}
     elif args.pack:
         pack = Path(args.pack)
     else:
@@ -596,6 +635,8 @@ def main() -> int:
         if not dat.roms:
             skipped += 1
             continue
+        if revision:
+            dat.name = f"{dat.name or member} @ {revision}"
         found = recipe_entries(dat, keep, targets)
         if not found:
             continue