chore: merge resolver and export fixes

scripts/common.py

@@ -430,11 +430,24 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
 
     # Each entry carries the system that declares it: a hashless name match is
     # narrowed to the maker folder that system names (resolve_local_file).
+    # A system that names its own cores (EmuELEC, AmberELEC, KNULLI) also
+    # passes them on, so another emulator's tree is foreign to its entries.
     # Fresh dicts, since a YAML anchor can share one entry between systems.
     config["systems"] = {
         sys_id: {
             **system,
-            "files": [{**f, "platform_system": sys_id} for f in system["files"]],
+            "files": [
+                {
+                    **f,
+                    "platform_system": sys_id,
+                    **(
+                        {"platform_cores": list(system["cores"])}
+                        if system.get("cores")
+                        else {}
+                    ),
+                }
+                for f in system["files"]
+            ],
         }
         if isinstance(system, dict) and system.get("files")
         else system
@@ -1754,6 +1767,24 @@ def resolve_local_file(
                             continue
                         if os.path.exists(path) and path not in candidates:
                             candidates.append(path)
+        # A platform entry names no profile, but a system that lists its
+        # cores says whose files it asks for. A name only another emulator's
+        # tree answers is no evidence, and the data directories hold those
+        # emulators' trees too: EmuELEC's VVVVVV data.zip was served xrick's
+        # archive of the same name, then xrick's data directory copy.
+        core_owners = {
+            token
+            for core in file_entry.get("platform_cores") or ()
+            for token in _affinity_tokens(str(core))
+        }
+        if (
+            not owners
+            and core_owners
+            and candidates
+            and all(_foreign_tree(p, core_owners) for p in candidates)
+        ):
+            candidates = []
+            unsourceable = True
         if makers and candidates:
             candidates = [p for p in candidates if not other_maker(p, makers, db)]
             if not candidates:

scripts/cross_reference.py

@@ -48,6 +48,7 @@ from common import (
     select_emulator_profiles,
 )
 from hashing import parse_hash_list
+from launchers import LauncherLayer, declared_destinations, held_record
 from validation import (
     case_folded_slots,
     declaration_settles,
@@ -692,6 +693,7 @@ SET_ASIDE = {
     "other_versions": "other versions of a slot",
     "not_read": "not read on this platform",
     "outside_scope": "outside the pack",
+    "launcher": "filled by the platform's launcher",
 }
 
 
@@ -923,6 +925,19 @@ def _cross_reference_profile(
         }
         if save_root:
             entry["root"] = "save"
+        elif index.get("launcher_candidates") is not None:
+            # What verify hands the launcher layer for the same entry.
+            probe = {
+                **f,
+                "profile": emu_name,
+                "emulator": profile.get("emulator", emu_name),
+                "path": effective_path,
+                "in_repo": in_repo,
+                "bundled": entry["bundled"],
+            }
+            index["launcher_candidates"].append(
+                (emu_name, is_standalone, f, probe, entry)
+            )
         gaps.append(entry)
 
     for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
@@ -934,12 +949,7 @@ def _cross_reference_profile(
         "systems": systems,
         "total_files": len(emu_files),
         "platform_covered": len(covered),
-        "gaps": len(gaps),
-        "gap_in_repo": sum(1 for g in gaps if g["in_repo"]),
-        "gap_missing": sum(1 for g in gaps if g["source"] == "missing"),
-        "gap_bios": sum(1 for g in gaps if g["source"] == "bios"),
-        "gap_data": sum(1 for g in gaps if g["source"] == "data"),
-        "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"),
+        **_gap_counts(gaps),
         "gap_details": gaps,
         "unsourceable": unsourceable_list,
         # Entries, not rows: an archive's members are one gap row.
@@ -950,6 +960,61 @@ def _cross_reference_profile(
     }
 
 
+def _gap_counts(gaps: list[dict]) -> dict[str, int]:
+    """The per-source tallies of a profile's gap rows."""
+    return {
+        "gaps": len(gaps),
+        "gap_in_repo": sum(1 for g in gaps if g["in_repo"]),
+        "gap_missing": sum(1 for g in gaps if g["source"] == "missing"),
+        "gap_bios": sum(1 for g in gaps if g["source"] == "bios"),
+        "gap_data": sum(1 for g in gaps if g["source"] == "data"),
+        "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"),
+    }
+
+
+def _settle_launcher_places(
+    report: dict,
+    candidates: list[tuple[str, bool, dict, dict, dict]],
+    layer: LauncherLayer,
+    db: dict,
+) -> None:
+    """Apply the platform's launcher places as find_undeclared_files does.
+
+    Batocera's DuckStation generator fills the PAL slot with scph5502.bin,
+    yet the report listed duckstation's scph1002a.bin as a file the pack
+    carries, which neither the pack nor the manifest does.
+    """
+    added, settled = layer.settle(
+        [(emu, standalone, f, probe) for emu, standalone, f, probe, _ in candidates],
+        lambda probe: held_record(probe, db),
+    )
+    for emu, _standalone, _f, probe, gap in candidates:
+        if id(probe) not in settled:
+            continue
+        data = report[emu]
+        data["gap_details"] = [g for g in data["gap_details"] if g is not gap]
+        data["undeclared_files"] -= 1
+        data["set_aside"]["launcher"] += 1
+    for entry in added:
+        data = report[entry["profile"]]
+        data["gap_details"].append({
+            "name": entry["name"],
+            "path": entry["path"],
+            "required": entry.get("required", False),
+            "bundled": False,
+            "note": "",
+            "source_ref": "",
+            "in_platform": False,
+            "in_repo": True,
+            "source": "bios",
+            "identity": entry.get("sha1") or "",
+            "launcher": entry["launcher"],
+            "launcher_of": entry["launcher_of"],
+        })
+    for data in report.values():
+        data.update(_gap_counts(data["gap_details"]))
+
+
 def cross_reference(
     profiles: dict[str, dict],
     declared: dict[str, set[str]],
@@ -960,6 +1025,7 @@ def cross_reference(
     standalone_cores: set[str] | None = None,
     folded_slots: frozenset[str] = frozenset(),
     platform: str = "",
+    launcher_layer: LauncherLayer | None = None,
 ) -> dict:
     """Compare emulator profiles against platform declarations.
 
@@ -998,12 +1064,15 @@ def cross_reference(
         "all_declared": all_declared,
         "folded_slots": folded_slots,
         "bios_dir": db.get("bios_dir"),
+        "launcher_candidates": [] if launcher_layer else None,
     }
     for emu_name, profile in profiles.items():
         _cross_reference_profile(
             emu_name, profile, declared, report, index, standalone_cores,
             platform,
         )
+    if launcher_layer:
+        _settle_launcher_places(report, index["launcher_candidates"], launcher_layer, db)
 
     return report
 
@@ -1195,6 +1264,10 @@ def main() -> None:
         standalone_cores=standalone_cores,
         folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
         platform=config.get("_key", args.platform) if args.platform else "",
+        # What the platform's launchers open, as verify and the builder read it.
+        launcher_layer=LauncherLayer(
+            config, profiles, relevant, db, declared_destinations(config)
+        ) if args.platform else None,
     )
 
     if args.json:

scripts/export_native.py

@@ -105,6 +105,25 @@ def _read_archive(unpack: Callable[[bytes], dict[str, str]], payload: bytes) ->
         ) from exc
 
 
+def _read_text(payload: bytes) -> str:
+    """A text original, refused when it cannot be the platform's file.
+
+    An empty body or an HTML error page served with 200 parsed as a file
+    with no entries, and four exporters rebuilt the platform's file from our
+    data alone and said OK: libretro's quoted names and Recalbox's comments
+    were gone.
+    """
+    text = payload.decode("utf-8", errors="replace")
+    head = text.lstrip()[:64].lower()
+    if not head:
+        raise OriginalUnreadable("empty file: --refresh-cache downloads it again")
+    if head.startswith(("<!doctype html", "<html")):
+        raise OriginalUnreadable(
+            "an HTML page, not the platform's file: --refresh-cache downloads it again"
+        )
+    return text
+
+
 def fetch(
     url: str,
     destination: Path,
@@ -331,7 +350,7 @@ def collect_originals(
     for relative, url in wanted.items():
         path = root / relative
         archive = callable(unpack) and relative.endswith((".zip", ".tar.gz"))
-        check = (lambda raw: _read_archive(unpack, raw)) if archive else None
+        check = (lambda raw: _read_archive(unpack, raw)) if archive else _read_text
         try:
             if allow_fetch:
                 payload = fetch(url, path, index, refresh, check)
@@ -348,7 +367,7 @@ def collect_originals(
                 if callable(keep):
                     keep(relative, payload)
             else:
-                originals[relative] = payload.decode("utf-8", errors="replace")
+                originals[relative] = _read_text(payload)
         # One platform's unreadable file fails that platform: raised, it
         # stopped --all before the platforms after it.
         except (OSError, http.client.HTTPException, OriginalUnreadable) as exc:
@@ -699,6 +718,8 @@ def export_platform(
     )
     if missing:
         return False, [f"the platform's own file is required: {m}" for m in missing]
+    if exporter.needs_original() and not originals:
+        return False, ["the platform's own file is required and none was read"]
 
     try:
         produced = exporter.render(systems, report, originals, scraped)
@@ -776,6 +797,8 @@ def export_platform(
             # A content-scanning core's names are labels: a list checked
             # by path cannot hold them.
             (report.labels, "entry", "entries", "found by content"),
+            (len(report.unattributed), "file", "",
+             "no single native system of the platform takes"),
         )
         if number
     ]
@@ -833,6 +856,7 @@ def export_platform(
         "requirements_corrected": requirements,
         "withheld": list(withheld),
         "found_by_content": report.labels,
+        "unattributed": report.unattributed,
         "removal_candidates": candidates,
         "doubled": doubled,
         "invalid": issues,

scripts/exporter/baseline.py

@@ -148,8 +148,10 @@ class NativeFile:
     # platform gives it), empty when no core the platform runs is named.
     runners: list[tuple[bool, str]] = field(default_factory=list)
     # The platform reads this file's md5 as a composite of its members
-    # (Recalbox on a .zip): the truth's md5 describes the container and
-    # can be neither compared with it nor written in its place.
+    # (Recalbox on a .zip) or as one member's (Batocera's zippedFile, which
+    # checkInsideZip compares inside the archive): the truth's md5 describes
+    # the container and can be neither compared with it nor written in its
+    # place.
     composite_md5: bool = False
     # Every truth entry of the system that answers to this platform entry's
     # name, whichever one the pairing took: the element names its cores, and
@@ -356,6 +358,9 @@ class Report:
     required_corrected: list[str] = field(default_factory=list)
     # Entries only content-scanning cores name: never written as files.
     labels: int = 0
+    # Additions to a system spanning several native ids that their cores do
+    # not tell apart, as truth system/name: not written.
+    unattributed: list[str] = field(default_factory=list)
 
     @property
     def corrections(self) -> int:
@@ -528,7 +533,8 @@ def build_native_model(
                     destination=destination,
                     native_system=native_id,
                     platform=file_entry,
-                    composite_md5=file_entry.get("zip_digest") == "composite",
+                    composite_md5=file_entry.get("zip_digest") == "composite"
+                    or bool(file_entry.get("zipped_file")),
                 )
             )
             report.files_kept += 1
@@ -551,7 +557,9 @@ def build_native_model(
     for system in systems.values():
         native_by_norm.setdefault(_norm_system_id(system.native_id), system.native_id)
 
-    def _target_native_ids(truth_sid: str) -> list[str]:
+    def _target_native_ids(truth_sid: str) -> tuple[list[str], bool]:
+        """The native ids a truth system spans, and whether the first one
+        is where an addition belongs without asking which core reads it."""
         scraped_key = truth_sid if truth_sid in scraped_systems else None
         if scraped_key is None:
             scraped_key = norm_to_scraped.get(_norm_system_id(truth_sid))
@@ -562,7 +570,7 @@ def build_native_model(
                 for fe in sys_data.get("files", [])
             }
             if not ids:
-                return [sys_data.get("native_id") or scraped_key]
+                return [sys_data.get("native_id") or scraped_key], True
             # A file the platform does not declare joins the system's primary
             # id, not whichever of its native ids sorts first: Recalbox files
             # pcengine, pcenginecd and supergrafx under one slug, and an
@@ -572,9 +580,9 @@ def build_native_model(
             if primary in ids:
                 ordered.remove(primary)
                 ordered.insert(0, primary)
-            return ordered
+            return ordered, primary in ids or len(ordered) == 1
         direct = native_by_norm.get(_norm_system_id(truth_sid))
-        return [direct] if direct else [truth_sid]
+        return ([direct] if direct else [truth_sid]), True
 
     # Pass 2: apply the truth onto that grouping.
     for truth_sid in sorted(truth.get("systems", {})):
@@ -583,7 +591,7 @@ def build_native_model(
         if not truth_files:
             continue
 
-        target_ids = _target_native_ids(truth_sid)
+        target_ids, settled_id = _target_native_ids(truth_sid)
         truth_files = [
             entry for entry in truth_files
             if (name := str(entry.get("name", "")))
@@ -596,6 +604,13 @@ def build_native_model(
             if candidate.truth is None
         ]
         pairs = pair_entries(truth_files, [c.platform or {} for c in candidates])
+        # The cores that read what the platform files under each native id,
+        # from the truth entries its declarations pair with.
+        readers: dict[str, set[str]] = {}
+        for t_index, index in pairs.items():
+            readers.setdefault(candidates[index].native_system, set()).update(
+                truth_files[t_index].get("_cores") or ()
+            )
         for candidate in (
             c
             for native_id in target_ids
@@ -660,8 +675,21 @@ def build_native_model(
             if platform_declares(truth_entry, declared):
                 continue
 
-            # The truth knows a file the platform does not declare.
+            # The truth knows a file the platform does not declare. Across
+            # several native ids and no primary, it joins the one whose files
+            # its cores read: RetroDECK's arcade spans mame and fbneo, and
+            # MAME device archives were filed as FBNeo BIOS because fbneo
+            # sorts first. With no single answer it is not written.
             native_id = target_ids[0]
+            if not settled_id:
+                wanted = set(truth_entry.get("_cores") or ())
+                shared = {nid: len(readers.get(nid, set()) & wanted) for nid in target_ids}
+                best = max(shared.values())
+                winners = [nid for nid, count in shared.items() if count == best]
+                if not best or len(winners) > 1:
+                    report.unattributed.append(f"{truth_sid}/{name}")
+                    continue
+                native_id = winners[0]
             system = systems.get(native_id)
             if system is None:
                 system = NativeSystem(native_id=native_id, from_platform=False)

scripts/exporter/misterfpga_exporter.py

@@ -83,6 +83,10 @@ class Exporter(BaseExporter):
         written = getattr(self, "_written", None)
         if written is not None and id(fe) not in written:
             return False
+        if field_name == "size" and not fe.size():
+            # The render writes a size only when the truth states one, and
+            # otherwise keeps MiSTer's beside the hash: nothing is removed.
+            return False
         return super().states(fe, field_name)
 
     def written(self, fe: NativeFile, field_name: str) -> str:

scripts/generate_site.py

@@ -1035,13 +1035,18 @@ def _split_contract(config: dict) -> tuple[dict, dict]:
     """
     allowed = _contract_keys()
     contract = {key: value for key, value in config.items() if key in allowed}
-    # platform_system is the resolver's copy of the key each entry sits under.
+    # platform_system and platform_cores are the resolver's copies of the key
+    # each entry sits under and of that system's cores.
     if isinstance(contract.get("systems"), dict):
         contract["systems"] = {
             sys_id: {
                 **system,
                 "files": [
-                    {k: v for k, v in f.items() if k != "platform_system"}
+                    {
+                        k: v
+                        for k, v in f.items()
+                        if k not in ("platform_system", "platform_cores")
+                    }
                     for f in system["files"]
                 ],
             }

scripts/launchers.py

@@ -20,7 +20,7 @@ import re
 from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 
-from common import declared_root, name_match_size_ok, sanitize_pack_path
+from common import declared_root, name_match_size_ok, resolve_local_file, sanitize_pack_path
 
 # The kinds whose names say which file the launcher opens. A link of a named
 # file is one too: ROCKNIX links /storage/roms/bios/nds_bios_arm9.bin into
@@ -330,6 +330,17 @@ def standalone_root(
     return next(iter(folders)) if len(folders) == 1 else None
 
 
+def held_record(report: dict, db: dict) -> dict | None:
+    """The database record of the file the collection holds for a report entry."""
+    local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "")
+    if local is None or status in ("not_found", "hash_mismatch"):
+        return None
+    for record in db.get("files", {}).values():
+        if record.get("path") == local:
+            return record
+    return None
+
+
 def declared_destinations(config: dict) -> set[str]:
     """Each destination the platform declares, relative to the core file directory.
 

scripts/packextras.py

@@ -555,7 +555,21 @@ def _agnostic_scan_extras(
                 if bios_dir and "/" in fpath and not fpath.startswith(bios_dir):
                     continue
 
-                path_prefix = resolved_dirs.get(id(f), "")
+                # A companion is read under its image's name with the
+                # extension the code appends, in that case: only a file of
+                # that extension is one, spelled as the code spells it. Its
+                # size identifies nothing, PCSX2 reads min(size, filesize)
+                # (BiosTools.cpp:234), and a 4 MB bound swept every smaller
+                # file of the folder to the BIOS root. It sits beside its
+                # image, so the scan looks where the image resolves: the
+                # held ps2-0230a-20080220.rom1 is a DVD firmware dump filed
+                # under DVD Firmware/.
+                companion_ext, image = _companion_of(profile, fname)
+
+                path_prefix = (
+                    resolved_dirs.get(id(image), "") if image
+                    else resolved_dirs.get(id(f), "")
+                )
                 if not path_prefix:
                     continue
                 # A name matching one file identifies it. A name matching several
@@ -570,11 +584,16 @@ def _agnostic_scan_extras(
                     path = entry.get("path", "")
                     if not path.startswith(path_prefix):
                         continue
-                    if not size_fits(f, entry.get("size", 0)):
+                    if not companion_ext and not size_fits(f, entry.get("size", 0)):
                         continue
                     scan_name = entry.get("name", "")
                     if not scan_name:
                         continue
+                    if companion_ext:
+                        stem, _, ext = scan_name.rpartition(".")
+                        if not stem or ext.lower() != companion_ext.lower():
+                            continue
+                        scan_name = f"{stem}.{companion_ext}"
                     # Into the directory the seed entry names, not the root:
                     # yaps2 reads its BIOS from pcsx2/bios/.
                     seed = f.get("path") or ""
@@ -771,6 +790,27 @@ def _companion_extras(
     return extras
 
 
+def _companion_of(profile: dict, name: str) -> tuple[str, dict | None]:
+    """The extension and the image of an entry named after a content-found image.
+
+    ps2-0230a-20080220.rom1 beside ps2-0230a-20080220.bin (found_by:
+    content) is the image's companion, read as <bios>.rom1. Empty for any
+    other entry.
+    """
+    stem, _, ext = name.rpartition(".")
+    if not stem:
+        return "", None
+    for entry in profile.get("files", []):
+        other = str(entry.get("name", ""))
+        if (
+            other != name
+            and entry.get("found_by") == "content"
+            and other.rpartition(".")[0].lower() == stem.lower()
+        ):
+            return ext, entry
+    return "", None
+
+
 def _companion_extensions(profile: dict, seed_name: str) -> set[str]:
     """Extensions the profile declares beside a free-named BIOS entry.
 
@@ -779,6 +819,8 @@ def _companion_extensions(profile: dict, seed_name: str) -> set[str]:
     CDVD.cpp:160-195). A profile states that by naming the companions after
     its BIOS entry: ps2-0230a-20080220.rom1 beside ps2-0230a-20080220.bin.
     """
+    if _companion_of(profile, seed_name)[0]:
+        return set()
     stem = seed_name.rsplit(".", 1)[0].lower()
     exts: set[str] = set()
     for entry in profile.get("files", []):
@@ -791,17 +833,23 @@ def _companion_extensions(profile: dict, seed_name: str) -> set[str]:
 def _companions(
     image_path: str, image_name: str, extensions: set[str], files_db: dict
 ) -> list[tuple[str, dict]]:
-    """Files beside a scanned image that the core opens as its companions."""
+    """Files beside a scanned image that the core opens as its companions.
+
+    Each is named as the core spells it: PCSX2 stats `<bios>.rom1`, then the
+    BIOS path with its extension replaced by `rom1`, both case-sensitive on
+    Linux (BiosTools.cpp:214-229). SCPH-70004.ROM1 as stored is never found
+    beside SCPH-70004.BIN, SCPH-70004.rom1 is.
+    """
     if not extensions:
         return []
     directory = image_path.rsplit("/", 1)[0] + "/"
-    stem = image_name.rsplit(".", 1)[0].lower()
-    wanted = {f"{image_name.lower()}.{ext}" for ext in extensions}
-    wanted |= {f"{stem}.{ext}" for ext in extensions}
+    stem = image_name.rsplit(".", 1)[0]
+    spelled = {f"{image_name}.{ext}".lower(): f"{image_name}.{ext}" for ext in extensions}
+    spelled.update({f"{stem}.{ext}".lower(): f"{stem}.{ext}" for ext in extensions})
     return sorted(
-        (sha1, entry)
+        (sha1, {**entry, "name": spelled[str(entry.get("name", "")).lower()]})
         for sha1, entry in _files_by_directory(files_db).get(directory, ())
-        if str(entry.get("name", "")).lower() in wanted
+        if str(entry.get("name", "")).lower() in spelled
     )
 
 

scripts/profile_sync.py

@@ -1154,11 +1154,19 @@ def select_views(
         branch = declared if pin_field == "source" else None
         head = None
         try:
+            unread: upstream.NotCachedError | None = None
             for candidate in branch_candidates(url):
-                head = upstream.resolve_head(repo, cache_dir, offline, candidate)
+                try:
+                    head = upstream.resolve_head(repo, cache_dir, offline, candidate)
+                except upstream.NotCachedError as exc:
+                    # Another prefix of the /tree/ tail may be the cached branch.
+                    unread = exc
+                    continue
                 if head:
                     branch = candidate
                     break
+            if unread and not head:
+                raise unread
             pin, origin = resolve_pin(
                 profile, repo, cache_dir, offline, pin_field, mode, branch
             )

scripts/provenance_report.py

@@ -80,11 +80,17 @@ def profile_needs(profiles: dict, db: dict) -> dict:
         (_crc_key(str(f.get("crc32", "")).lower()), f.get("size"))
         for f in files.values() if f.get("crc32")
     }
+    # Held under its own name or under an alias the database indexes, as
+    # the resolver and verify read it: np2pi's KANJI2.ROM is the PC-88
+    # n88knj2.rom, and its own name alone listed it missing.
     sizes_by_name: dict[str, set[int]] = {}
     for record in files.values():
         sizes_by_name.setdefault(str(record.get("name", "")).casefold(), set()).add(
             record.get("size")
         )
+    for alias, sha1s in indexes.get("by_name", {}).items():
+        held = sizes_by_name.setdefault(str(alias).casefold(), set())
+        held.update(files[sha1].get("size") for sha1 in sha1s if sha1 in files)
 
     by_hash: dict[tuple, list[dict]] = {}
     by_name: dict[str, list[tuple[dict, dict]]] = {}
@@ -117,7 +123,9 @@ def profile_needs(profiles: dict, db: dict) -> dict:
             if all(entry.get(k) is None for k in ("size", "min_size", "max_size")):
                 continue
             names = [name, *(entry.get("aliases") or [])]
-            if any(
+            # An unsourceable entry is not held by a homonym: xm7's
+            # KANJI2.ROM is the Fujitsu font, the held one the PC-88's.
+            if not entry.get("unsourceable") and any(
                 _fits(held, entry)
                 for alias in names
                 for held in sizes_by_name.get(str(alias).casefold(), ())

scripts/scraper/retropie_scraper.py

@@ -56,6 +56,18 @@ _VERSION = re.compile(r'^__version="([^"]+)"', re.MULTILINE)
 _PACKAGE_DIRS = _SCRIPT_SECTIONS
 
 
+def core_name(module_id: str) -> str:
+    """The core a RetroPie package installs, as the buildbot names it.
+
+    A libretro package is lr-<core> with the name hyphenated (lr-beetle-psx
+    builds beetle_psx). A standalone package keeps its id (dosbox-staging).
+    The platform and target scrapers both read it here.
+    """
+    if module_id.startswith("lr-"):
+        return module_id[3:].replace("-", "_")
+    return module_id
+
+
 class Scraper(BaseScraper):
     """Scraper for the RetroPie package list."""
 
@@ -148,10 +160,11 @@ class Scraper(BaseScraper):
         libretro cores and reads the same files, at BIOS/ instead of
         system/. Only the core list is its own.
         """
-        # A libretro package is lr-<core>; a standalone package is named
-        # after the emulator itself.
+        # Named by core_name(), as the target scraper names them. Kept
+        # hyphenated, lr-dosbox-pure named dosbox-pure, the standalone
+        # DOSBox Pure Unleashed build RetroPie does not ship.
         modules = self.module_ids()
-        cores = sorted({module.removeprefix("lr-") for module in modules})
+        cores = sorted({core_name(m) for m in modules})
         libretro = {m.removeprefix("lr-") for m in modules if m.startswith("lr-")}
         # The retroarch package is the frontend that runs the lr- cores.
         standalone = sorted(

scripts/scraper/targets/retropie_targets_scraper.py

@@ -19,6 +19,7 @@ from datetime import datetime, timezone
 
 import yaml
 
+from ..retropie_scraper import core_name
 from . import BaseTargetScraper
 
 PLATFORM_NAME = "retropie"
@@ -158,15 +159,10 @@ class Scraper(BaseTargetScraper):
             if not module_id:
                 print(f"  warning: no rp_module_id in {filename}", file=sys.stderr)
                 continue
-            # A libretro package is lr-<core> with the buildbot name hyphenated
-            # (lr-beetle-psx -> beetle_psx), and a standalone package keeps the id
-            # the platform list carries (dosbox-staging).
-            core_name = module_id
-            if core_name.startswith("lr-"):
-                core_name = core_name[3:].replace("-", "_")
+            # The platform scraper names the package the same way.
             for platform in PLATFORM_FLAGS:
                 if _is_available(flags, platform):
-                    platform_cores[platform].add(core_name)
+                    platform_cores[platform].add(core_name(module_id))
 
         print(f"  parsed {len(filenames)} scriptmodules", file=sys.stderr)
 

scripts/scraper/targets/rocknix_targets_scraper.py

@@ -30,8 +30,10 @@ DEVICES_DIR = "projects/ROCKNIX/devices"
 
 try:
     from ..base_scraper import fetch_bytes, github_headers
+    from ..rocknix_scraper import standalone_emulators
 except ImportError:
     from scraper.base_scraper import fetch_bytes, github_headers
+    from scraper.rocknix_scraper import standalone_emulators
 
 # `NAME="a b \
 #   c"` or `NAME+=" a b"`, continuation lines joined first.
@@ -91,11 +93,13 @@ def core_names(libretro: list[str], standalone: list[str]) -> list[str]:
     """Package names as the profiles' cores: lists spell them.
 
     A libretro package is the core's name with `-lr` (`beetle-psx-lr`,
-    `neocd_lr`), spelled with underscores as the buildbot names it, and a
-    standalone package drops its `-sa` (`duckstation-sa`, `hatarisa`).
+    `neocd_lr`), spelled with underscores as the buildbot names it. A
+    standalone package is kept under both spellings, as the platform
+    scraper keeps it: `vice-sa` is the only name vice-standalone answers
+    to, `vice` that of the libretro profile.
     """
     names = {re.sub(r"[-_]lr$", "", package).replace("-", "_") for package in libretro}
-    names.update(re.sub(r"-?sa$", "", package) for package in standalone)
+    names.update(standalone_emulators(standalone))
     return sorted(names)
 
 

scripts/slots.py

@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING
 
 import nativemode
 from common import (
+    compute_hashes,
     platform_arg,
     archive_entry,
     build_zip_contents_index,
@@ -759,12 +760,18 @@ def find_collisions(
 def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool:
     """Whether the two layers agree on what ships at this destination.
 
-    They agree when they resolve to the same file, when the platform's own
+    They agree when they resolve to the same bytes, when the platform's own
     hash list accepts the profile's file, or when the profile's accepts the
     platform's. A contradiction needs both lists to reject the other side.
     """
     if profile.local_path == platform.local_path:
         return True
+    # Two paths can hold one content: AmberELEC's bluemsx declarations
+    # resolve into the data/bluemsx cache, the profile's into bios/.
+    if profile.local_path and platform.local_path:
+        shipped = _content_sha1(platform.local_path, db)
+        if shipped is not None and shipped == _content_sha1(profile.local_path, db):
+            return True
     members = profile.entry.get("members")
     if members and platform.local_path:
         return holds_required_members(platform.local_path, members)
@@ -821,6 +828,19 @@ def _record_at(path: str, db: dict) -> dict | None:
 _PATH_INDEX: dict[int, tuple[dict, dict[str, dict]]] = {}
 
 
+def _content_sha1(path: str, db: dict) -> str | None:
+    """The sha1 of the file at path, from the database or read from disk.
+
+    A data directory cache is not in the database, so its files are hashed.
+    """
+    record = _record_at(path, db)
+    if record is not None:
+        return record.get("sha1")
+    if not os.path.isfile(path):
+        return None
+    return compute_hashes(path, frozenset({"sha1"}))["sha1"]
+
+
 def _accepted_by_all(path: str, entries: list[dict], db: dict) -> bool:
     """Whether every declaration's own hashes accept the file at path."""
     record = _record_at(path, db)

scripts/upstream.py

@@ -432,6 +432,22 @@ def fetch_file(
 MOVING_TTL = 6 * 3600
 
 
+def _api_target(url: str, cache_dir: str) -> Path:
+    """Where the cached answer to one API call lives."""
+    key = hashlib.sha256(url.encode()).hexdigest()
+    return Path(cache_dir) / "_api" / f"{key}.json"
+
+
+def _require_cached(url: str, cache_dir: str, offline: bool, what: str) -> None:
+    """Offline, refuse a call the cache holds no answer to, absence included.
+
+    _api answers None both for a resource the forge does not have and for
+    one never asked, and only the first is a fact.
+    """
+    if offline and not _api_target(url, cache_dir).is_file():
+        raise NotCachedError(f"{what} is not in the offline cache")
+
+
 def _api(
     url: str, cache_dir: str, offline: bool, max_age: float | None = None,
     absent_max_age: float | None = None,
@@ -443,8 +459,7 @@ def _api(
     ``absent_max_age`` bounds a cached absence alone, for a resource that
     can appear but, once there, does not change.
     """
-    key = hashlib.sha256(url.encode()).hexdigest()
-    target = Path(cache_dir) / "_api" / f"{key}.json"
+    target = _api_target(url, cache_dir)
     if target.is_file():
         raw = target.read_text(encoding="utf-8")
         bound = (
@@ -542,19 +557,31 @@ def resolve_head(
     default branch, so the code a profile documents can be absent from the
     tip the forge serves by default. ``fresh`` asks the forge again whatever
     the cache holds.
+
+    Offline, a tip the cache never stored raises NotCachedError: returning
+    None read as a branch the forge does not have, and profile_sync dropped
+    xenios-jp/XeniOS, judging its citations against the other repository.
     """
-    return _first_sha(
-        _api(_commits_url(repo, None, branch), cache_dir, offline, 0 if fresh else MOVING_TTL)
+    url = _commits_url(repo, None, branch)
+    _require_cached(
+        url, cache_dir, offline, f"{repo.slug}: tip of {branch or 'the default branch'}"
     )
+    return _first_sha(_api(url, cache_dir, offline, 0 if fresh else MOVING_TTL))
 
 
 def resolve_commit_at(
     repo: Repo, date: str, cache_dir: str, offline: bool = False,
     branch: str | None = None,
 ) -> str | None:
-    """Last commit on or before a date, on the default branch or a named one."""
+    """Last commit on or before a date, on the default branch or a named one.
+
+    Offline, an answer the cache never stored raises NotCachedError, as a
+    branch tip does: None means the forge has no such commit.
+    """
     if repo.family in ("github", "gitlab"):
-        return _first_sha(_api(_commits_url(repo, date, branch), cache_dir, offline))
+        url = _commits_url(repo, date, branch)
+        _require_cached(url, cache_dir, offline, f"{repo.slug}: commit at {date}")
+        return _first_sha(_api(url, cache_dir, offline))
     return _gitea_commit_at(repo, date, cache_dir, offline, branch)
 
 
@@ -581,7 +608,9 @@ def _gitea_commit_at(
         base += f"&sha={urllib.parse.quote(branch, safe='')}"
     page = 1
     while True:
-        payload = _api(f"{base}&page={page}", cache_dir, offline, MOVING_TTL)
+        url = f"{base}&page={page}"
+        _require_cached(url, cache_dir, offline, f"{repo.slug}: commit at {date}")
+        payload = _api(url, cache_dir, offline, MOVING_TTL)
         if not isinstance(payload, list) or not payload:
             return None
         for entry in payload:

scripts/verify.py

@@ -85,7 +85,7 @@ from common import (
 
 yaml = require_yaml()
 from exporter.candidates import BIOS, EntryCategories
-from launchers import LauncherLayer, declared_destinations
+from launchers import LauncherLayer, declared_destinations, held_record
 from nativemode import (
     SIZE_AND_ANY_HASH,
     digest_algorithm,
@@ -490,17 +490,6 @@ def _identity_of(entry: dict) -> dict:
     }
 
 
-def _held_record(report: dict, db: dict) -> dict | None:
-    """The database record of the file the collection holds for a report entry."""
-    local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "")
-    if local is None or status in ("not_found", "hash_mismatch"):
-        return None
-    for record in db.get("files", {}).values():
-        if record.get("path") == local:
-            return record
-    return None
-
-
 def find_undeclared_files(
     config: dict,
     emulators_dir: str,
@@ -779,7 +768,7 @@ def find_undeclared_files(
 
     if launcher_layer:
         added, settled = launcher_layer.settle(
-            launcher_candidates, lambda report: _held_record(report, db)
+            launcher_candidates, lambda report: held_record(report, db)
         )
         undeclared = [u for u in undeclared if id(u) not in settled]
         undeclared.extend(added)