chore: merge resolver fixes, fourth pass

scripts/exporter/batocera_exporter.py

@@ -19,7 +19,9 @@ SOURCE_URL = (
     "/package/batocera/core/batocera-scripts/scripts/batocera-systems"
 )
 
-_ENTRY_START = re.compile(r'^(\s{4})"([^"]+)":\s*\{')
+# Any indent: Batocera writes four spaces, EmuELEC one tab for 23 systems,
+# and an entry the pattern missed was appended again whole.
+_ENTRY_START = re.compile(r'^(\s+)"([^"]+)":\s*\{')
 _INDENT = " " * 4
 # One BIOS item: a dict literal with no nested braces.
 _ITEM = re.compile(r"\{[^{}\n]*\}")
@@ -382,12 +384,13 @@ class Exporter(BaseExporter):
         return inner if isinstance(inner, dict) else {}
 
     def _entry_line(
-        self, system: NativeSystem, files: list[NativeFile], before: dict | None = None
+        self, system: NativeSystem, files: list[NativeFile], before: dict | None = None,
+        indent: str = _INDENT,
     ) -> str:
         own = self._own_entry(system, before)
         runner = "".join(f"{key}, " for key in self._runner_keys(own))
         return (
-            f'{_INDENT}"{system.native_id}": '
+            f'{indent}"{system.native_id}": '
             f'{{ "name": "{self.display_name(system)}", {runner}'
             f'"biosFiles": [ {self._bios_files(files, own)} ] }},'
         )
@@ -440,16 +443,17 @@ class Exporter(BaseExporter):
         before = self._parse_entry(original)
         own = self._own_entry(system, before)
         items = self._bios_items(files, own)
+        indent = _ENTRY_START.match(original[0]).group(1)
         one_line = len(original) == 1 and not trailing and not leading
         if one_line:
-            return [self._entry_line(system, files, before)]
+            return [self._entry_line(system, files, before, indent)]
 
         runner = "".join(f"{key}, " for key in self._runner_keys(own))
         head = (
-            f'{_INDENT}"{system.native_id}": '
+            f'{indent}"{system.native_id}": '
             f'{{ "name": "{self.display_name(system)}", {runner}"biosFiles": ['
         )
-        pad = " " * (len(_INDENT) + 4)
+        pad = indent + " " * 4
         lines = [head]
         for index, item in enumerate(items):
             key = _note_key(item)
@@ -458,7 +462,7 @@ class Exporter(BaseExporter):
             note = trailing.get(key, "")
             suffix = f" {note}" if note else ""
             lines.append(f"{pad}{item}{separator}{suffix}")
-        lines.append(f"{_INDENT}] }},")
+        lines.append(f"{indent}] }},")
         return lines
 
     def _patched_entry(

scripts/generate_db.py

@@ -576,12 +576,26 @@ def _collect_all_aliases(
     # naming its content once 64DD_IPL.bin sorted first, and its aliases
     # 64dd_ipl_J.bin and 64DD_IPL_JP.n64 went with it.
     holders: dict[str, set[str]] = {}
+    # The files a name designates once case is folded, as the resolver's
+    # name step folds it, a primary and its own .variants/ counted as one
+    # file. dosbox's SC-55 ROM2.BIN found Galaksija's ROM2.BIN the only
+    # exact holder, and its alias ROM2_ST.BIN went to a 4 KiB BASIC ROM,
+    # while SC-55's rom2.bin sat under the same folded name.
+    families: dict[str, set[str]] = {}
+
+    def _hold(name: str, path: str, sha1: str) -> None:
+        holders.setdefault(name, set()).add(sha1)
+        directory = path.rpartition("/")[0].removesuffix("/.variants")
+        families.setdefault(name.casefold(), set()).add(
+            f"{directory}/{variant_base_name(name)}".casefold()
+        )
+
     for sha1, entry in files.items():
-        holders.setdefault(entry["name"], set()).add(sha1)
+        _hold(entry["name"], entry["path"], sha1)
     for sha1, copy_list in (copies or {}).items():
         if sha1 in files:
             for copy in copy_list:
-                holders.setdefault(copy["name"], set()).add(sha1)
+                _hold(copy["name"], copy["path"], sha1)
 
     aliases = {}
 
@@ -691,11 +705,16 @@ def _collect_all_aliases(
                     ]
                     if taken:
                         shadowed.append((entry_name, set(matched), taken))
-                if not matched and len(holders.get(entry_name, ())) == 1:
+                held = holders.get(entry_name, set())
+                if (
+                    not matched
+                    and len(held) == 1
+                    and len(families[entry_name.casefold()]) == 1
+                ):
                     # A name carried by several contents names none of them:
                     # quasi88's disk.rom aliases went to whichever
                     # disk.rom the scan met last, a Tandy CoCo ROM.
-                    matched = set(holders[entry_name])
+                    matched = set(held)
                 for sha in sorted(matched):
                     for alias_name in entry_aliases:
                         _add_alias(alias_name, sha)

scripts/profile_sync.py

@@ -2665,6 +2665,9 @@ def drift_score(
     # set again before any ref can be read, so each outranks a GONE ref:
     # pcem ranked last at 0 with an orphaned upstream_commit.
     score += PIN_WEIGHT * (len(report.orphaned) + len(report.moved_repos))
+    # A profile no repository answers for cannot be read at all, and
+    # --strict fails on it: it ranked at 0 under the clean profiles.
+    score += PIN_WEIGHT * report.uncheckable()
     score += min(commits, 20) // 4
     return score
 
@@ -3742,15 +3745,6 @@ def _git_file_at(path: Path, sha: str) -> dict | None:
     return document if isinstance(document, dict) else None
 
 
-def _scalar_values(document: dict) -> dict[str, str]:
-    """String scalars of the document, keyed by field path."""
-    return {
-        field: value
-        for kind, field, _, _, value in _walk_document(document)
-        if kind == "str"
-    }
-
-
 def _pin_of(
     document: dict, field: str, mode: str = "",
     repo: upstream.Repo | None = None,
@@ -3788,26 +3782,6 @@ def _pin_of(
     return value
 
 
-_CITATION_RUN_RE = re.compile(
-    PROSE_CITE_RE.pattern + r"(?:, *\d+(?:-\d+)?)*"
-)
-
-
-def _scalar_citations(text: str | None) -> tuple[str, ...] | None:
-    """The citations a scalar carries, in order, without the prose around them.
-
-    A note reworded around unchanged citations was still written under the
-    pin those citations describe: comparing whole texts moved the writing
-    revision to the rewording commit, whose pin is current, and the stale
-    citations were never offered for realignment again.
-    """
-    if text is None:
-        return None
-    return tuple(
-        _citation_key(m.group(0)) for m in _CITATION_RUN_RE.finditer(text)
-    )
-
-
 def _writing_pins(
     revisions: list[tuple[str, dict]], intro_sha: str, field: str, mode: str = "",
     repo: upstream.Repo | None = None,
@@ -3943,7 +3917,8 @@ def _realign_part(
 
 
 def _writing_pairs(
-    document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str
+    document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str,
+    served: dict | None = None,
 ) -> tuple[list[list], list[str]]:
     """Every reading of where the text was written, and the ambiguities.
 
@@ -3973,8 +3948,9 @@ def _writing_pairs(
             ambiguous.append(f"{pin_field}_commit {' -> '.join(written)}")
         # The current pin among the candidates reads as "no move" (None) for
         # that repository, which the other candidates must then agree with.
+        reader = (served or {}).get(repo, repo)
         options.append([
-            (repo, pin, current) if pin != current else None for pin in written
+            (reader, pin, current) if pin != current else None for pin in written
         ])
     readings = [
         [pair for pair in combo if pair is not None]
@@ -4055,8 +4031,20 @@ def realign_prose(
     ]
     if not repos:
         return []
+    # A source declared gone is read through its mirror, which carries the
+    # same commits: teknoparrot's notes stayed blocked once
+    # teknogods/TeknoParrotUI answered 404, every reading of the old pin
+    # calling the files absent.
+    mirror = (
+        upstream.parse_repo(str(document.get("source_mirror") or ""))
+        if document.get("upstream_gone") else None
+    )
+    served = {
+        repo: mirror for field, _mode, repo in repos
+        if mirror is not None and field == "source"
+    }
     pinned = [
-        (repo, pin)
+        (served.get(repo, repo), pin)
         for field, mode, repo in repos
         if (pin := _pin_of(document, f"{field}_commit", mode))
     ]
@@ -4066,72 +4054,79 @@ def realign_prose(
     if not history:
         return []
 
-    current_values = _scalar_values(document)
-    fields = {citation.field for citation in citations}
-    alive = dict.fromkeys(fields, True)
-    intro: dict[str, str | None] = dict.fromkeys(fields)
+    # Each citation is followed on its own: a field's text changes whenever
+    # one of its citations is corrected by hand, and dating the whole field
+    # from that commit put idtech4a++'s untouched Game.java:69-72 under the
+    # current pin, so the stale range was never offered again.
+    keys = [(c.field, _citation_key(c.ref)) for c in citations]
+    alive = [True] * len(citations)
+    intro: list[str | None] = [None] * len(citations)
     revisions: list[tuple[str, dict]] = []
-    # Fields whose writing pin lies past a revision that cannot be read:
+    # Citations whose writing pin lies past a revision that cannot be read:
     # where their text came in is unknown, so they are not moved.
-    unknown: dict[str, str] = {}
+    unknown: dict[int, str] = {}
     # A revision that does not parse is passed over unless the pins differ
-    # on its two sides while the field's citations changed across it: then
-    # the text may have come in there under either pin. Otherwise nothing
+    # on its two sides while the citation is absent across it: then the
+    # text may have come in there under either pin. Otherwise nothing
     # it could hold changes the reading. lindbergh-loader
     # committed invalid YAML once (df7346b3), fixed the same day, and every
     # note older than that commit stayed blocked for good.
     gap: str | None = None
+
+    def carried_by(past: dict) -> set[tuple[str, str]]:
+        return {
+            (c.field, _citation_key(c.ref))
+            for c in collect_citations(past) if c.kind == "prose"
+        }
+
     for sha in history:
-        if not any(alive.values()):
+        if not any(alive):
             break
         past = _git_file_at(path, sha)
         if past is None:
             gap = gap or sha
             continue
+        held = carried_by(past)
         if gap:
             newer = revisions[-1][1] if revisions else document
             moved = any(
                 newer.get(pin) != past.get(pin)
                 for pin in ("source_commit", "upstream_commit")
             )
-            values = _scalar_values(past)
-            for field, live in alive.items():
-                if live and moved and _scalar_citations(values.get(field)) != (
-                    _scalar_citations(current_values.get(field))
-                ):
-                    unknown[field] = gap
-                    alive[field] = False
+            for index, live in enumerate(alive):
+                if live and moved and keys[index] not in held:
+                    unknown[index] = gap
+                    alive[index] = False
             gap = None
         revisions.append((sha, past))
-        values = _scalar_values(past)
-        for field in fields:
-            if not alive[field]:
+        for index, live in enumerate(alive):
+            if not live:
                 continue
-            if _scalar_citations(values.get(field)) == _scalar_citations(
-                current_values.get(field)
-            ):
-                intro[field] = sha
+            if keys[index] in held:
+                intro[index] = sha
             else:
-                alive[field] = False
+                alive[index] = False
     if gap:
         # The oldest revisions do not parse: where the text came in is past
         # what can be read.
-        unknown.update({field: gap for field, live in alive.items() if live})
+        unknown.update({index: gap for index, live in enumerate(alive) if live})
 
     messages: list[str] = []
     jobs: list[tuple[Citation, dict]] = []
-    for citation in citations:
-        if citation.field in unknown:
+    for index, citation in enumerate(citations):
+        if index in unknown:
             messages.append(
                 f"read again: {citation.field}: {citation.ref} "
-                f"(revision {unknown[citation.field][:8]} unreadable)"
+                f"(revision {unknown[index][:8]} unreadable)"
             )
             continue
-        intro_sha = intro.get(citation.field)
+        intro_sha = intro[index]
         if intro_sha is None:
             # The scalar is not committed yet: written now, under this pin.
             continue
-        readings, ambiguous = _writing_pairs(document, repos, revisions, intro_sha)
+        readings, ambiguous = _writing_pairs(
+            document, repos, revisions, intro_sha, served
+        )
         if not any(readings):
             continue
         moves, blocked = _realign_citation(
@@ -5042,7 +5037,7 @@ def _print_triage(
         state = triage_state(report)
         if refused:
             state += f", {refused}"
-        pending += bool(report.needs_review())
+        pending += bool(report.needs_review() or report.uncheckable())
         print(f"{score:5d}  {report.name:30s}  {state}")
     print(f"{len(ranked)} profiles, {pending} with something to review")
 

scripts/scraper/targets/amberelec_targets_scraper.py

@@ -0,0 +1,157 @@
+"""Scraper for AmberELEC per-device emulator availability.
+
+Sources (AmberELEC/AmberELEC, at the commit platforms/amberelec.yml names):
+  - Makefile -- one `DEVICE=<name> ARCH=<arch> ./scripts/build_distro` target
+    per image (RG351P, RG351V, RG351MP, RG552, RG353, RG503 at a21975bf)
+  - packages/amberelec/package.mk -- LIBRETRO_CORES_EXTRA joins PKG_EMUS
+    only on the devices its `if` names (lines 23-31 at a21975bf: RG552,
+    RG353, RG503), and on every other device makeinstall_target deletes
+    each of those cores from es_systems.cfg (lines 45-58)
+  - packages/ui/emulationstation/config/es_systems.cfg -- read the way the
+    platform scraper reads it, after that deletion
+
+An RG351P image carries neither melonDS nor bsnes: without a target list,
+its pack shipped the Nintendo DS firmware anyway.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from datetime import datetime, timezone
+
+import yaml
+
+from . import BaseTargetScraper
+
+PLATFORM_NAME = "amberelec"
+
+RAW_BASE = "https://raw.githubusercontent.com/AmberELEC/AmberELEC/{ref}"
+META_MK = "packages/amberelec/package.mk"
+
+try:
+    from ..amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS
+    from ..base_scraper import fetch_bytes
+    from ..batocera_scraper import _not_cores
+    from ..es_emulators import es_cores, xml_systems
+except ImportError:
+    from scraper.amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS
+    from scraper.base_scraper import fetch_bytes
+    from scraper.batocera_scraper import _not_cores
+    from scraper.es_emulators import es_cores, xml_systems
+
+_DEVICE_RE = re.compile(
+    r"^(\w+):\n\tDEVICE=(\w+) ARCH=(\w+) \./scripts/build_distro$", re.MULTILINE
+)
+_EXTRA_RE = re.compile(r'LIBRETRO_CORES_EXTRA="([^"]*)"')
+# The `if` whose body adds the extra cores to PKG_EMUS.
+_EXTRA_IF_RE = re.compile(
+    r"if \[\[ (?P<test>[^\n]*?) \]\]; then\s*\n\s*PKG_EMUS\+=\" \$\{LIBRETRO_CORES_EXTRA\}\""
+)
+# What makeinstall_target does to es_systems.cfg on the other devices.
+_REMOVAL_LINES = (
+    "for CORE in ${LIBRETRO_CORES_EXTRA}; do",
+    'sed -i "s|<core>$CORE</core>||g"',
+    r"sed -i -e '/<emulator name/{:a;N;/<\/emulator>/!ba;/melonds/d}'",
+)
+_EMULATOR_BLOCK_RE = re.compile(r"[^\n]*<emulator name.*?</emulator>[^\n]*\n?", re.DOTALL)
+
+
+def devices(makefile: str) -> dict[str, str]:
+    """{device: architecture} of each image the Makefile builds."""
+    found = {device: arch for target, device, arch in _DEVICE_RE.findall(makefile)
+             if target == device}
+    if not found:
+        raise RuntimeError("the AmberELEC Makefile builds no device image")
+    return found
+
+
+def extra_cores(package_mk: str) -> tuple[list[str], set[str]]:
+    """(the extra cores, the devices that keep them) from the metapackage."""
+    listed = _EXTRA_RE.search(package_mk)
+    test = _EXTRA_IF_RE.search(package_mk)
+    if listed is None or test is None:
+        raise RuntimeError(f"{META_MK}: no LIBRETRO_CORES_EXTRA or no test adding it")
+    missing = [line for line in _REMOVAL_LINES if line not in package_mk]
+    if missing:
+        raise RuntimeError(f"{META_MK}: makeinstall_target no longer reads {missing[0]!r}")
+    keep = set(re.findall(r'"\$\{DEVICE\}" == "(\w+)"', test.group("test")))
+    for alternatives in re.findall(r'"\$\{DEVICE\}" =~ \^\(([\w|]+)\)\$', test.group("test")):
+        keep.update(alternatives.split("|"))
+    if not keep:
+        raise RuntimeError(f"{META_MK}: the test adding the extra cores names no device")
+    return listed.group(1).split(), keep
+
+
+def without_cores(es_systems: str, cores: list[str]) -> str:
+    """es_systems.cfg as makeinstall_target leaves it on a device without them."""
+    for core in cores:
+        if core == "melonds":
+            es_systems = _EMULATOR_BLOCK_RE.sub(
+                lambda block: "" if "melonds" in block.group(0) else block.group(0),
+                es_systems,
+            )
+        es_systems = es_systems.replace(f"<core>{core}</core>", "")
+    return es_systems
+
+
+def device_cores(es_systems: str, extra: list[str], keeps_extra: bool) -> list[str]:
+    """Every core and standalone emulator one device's es_systems.cfg offers."""
+    text = es_systems if keeps_extra else without_cores(es_systems, extra)
+    cores, _standalone, _dual = es_cores(
+        xml_systems(text), RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME)
+    )
+    return sorted(set(cores))
+
+
+class Scraper(BaseTargetScraper):
+    """Per-device core lists from AmberELEC's metapackage."""
+
+    def __init__(self, url: str = "https://github.com/AmberELEC/AmberELEC") -> None:
+        super().__init__(url=url)
+        self.ref = ""
+
+    def _raw(self, path: str) -> str:
+        return fetch_bytes(f"{RAW_BASE.format(ref=self.ref)}/{path}").decode("utf-8")
+
+    def fetch_targets(self) -> dict:
+        self.ref = self.pinned_revision(PLATFORM_NAME)
+        extra, keep = extra_cores(self._raw(META_MK))
+        es_systems = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8")
+        targets: dict[str, dict] = {}
+        for device, arch in sorted(devices(self._raw("Makefile")).items()):
+            cores = device_cores(es_systems, extra, device in keep)
+            targets[device.lower()] = {"architecture": arch, "cores": cores}
+            print(f"  {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr)
+        return {
+            "platform": PLATFORM_NAME,
+            "source": f"{self.url}/blob/{self.ref}/{META_MK}",
+            "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "targets": targets,
+        }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False, description="Scrape AmberELEC per-device cores"
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Show target summary")
+    parser.add_argument("--output", "-o", help="Output YAML file")
+    args = parser.parse_args()
+
+    scraper = Scraper()
+    data = scraper.fetch_targets()
+    if args.dry_run:
+        for name, info in data["targets"].items():
+            print(f"  {name} ({info['architecture']}): {len(info['cores'])} cores")
+        return
+    if args.output:
+        scraper.write_output(data, args.output)
+        print(f"Written to {args.output}")
+        return
+    print(yaml.dump(data, default_flow_style=False, sort_keys=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/slots.py

@@ -43,7 +43,12 @@ from common import (
     variant_base_name,
 )
 from packresolve import preferred_entries
-from validation import outside_gap_scope, read_from_save_dir, with_archive_members
+from validation import (
+    content_alternatives,
+    outside_gap_scope,
+    read_from_save_dir,
+    with_archive_members,
+)
 
 if TYPE_CHECKING:
     from packextras import ExtraPlacement
@@ -85,6 +90,9 @@ class Claim:
     entry: dict = field(default_factory=dict)
     local_path: str | None = None
     status: str = ""
+    # A core that identifies its firmware by content reads no file by this
+    # name: any dump it takes for the same system and region answers for it.
+    accepts: tuple[dict, ...] = ()
 
     @property
     def is_proven(self) -> bool:
@@ -260,6 +268,7 @@ def profile_claims(
                 )
                 if status == "name_exact" and _size_the_code_reads(entry, local):
                     status = SIZE_EXACT
+                accepts = tuple(content_alternatives(entry, profile))
                 claims.append(
                     Claim(
                         origin="profile",
@@ -269,6 +278,7 @@ def profile_claims(
                         entry=entry,
                         local_path=local,
                         status=status,
+                        accepts=accepts,
                     )
                 )
     return claims
@@ -777,6 +787,10 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool:
     They agree when they resolve to the same bytes, when the platform's own
     hash list accepts the profile's file, or when the profile's accepts the
     platform's. A contradiction needs both lists to reject the other side.
+    A core found by content (found_by: content) reads no file by the
+    destination's name, so any dump it takes for the same system and region
+    settles the slot: DuckStation takes Batocera's scph1001.bin v2.0, which its profile
+    labels scph1001_v20.bin.
     """
     if profile.local_path == platform.local_path:
         return True
@@ -795,6 +809,11 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool:
         and _accepted_by_all(profile.local_path, [platform.entry], db)
     ):
         return True
+    if platform.local_path and any(
+        _lists_hashes(label) and _accepted_by_all(platform.local_path, [label], db)
+        for label in profile.accepts
+    ):
+        return True
     return bool(
         platform.local_path
         and _lists_hashes(profile.entry)

scripts/sourceref.py

@@ -220,10 +220,21 @@ PROSE_EXTENSIONLESS = (
     "COPYING",
     "CHANGELOG",
 )
-PROSE_CITE_RE = re.compile(
+# Any other dotless file is cited with its directory, which a constant or a
+# hex address never carries: lightspark's tools/errorgen:20.
+_PROSE_PATH = (
     r"(?P<path>[A-Za-z0-9_][\w./+-]*\.[A-Za-z]\w*"
-    r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r"))"
-    r":(?P<range>\d+(?:-\d+)?)"
+    r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r")"
+    r"|(?:[\w.+-]+/)+[A-Za-z_][\w+-]*)"
+)
+PROSE_CITE_RE = re.compile(_PROSE_PATH + r":(?P<range>\d+(?:-\d+)?)")
+# A file named before its locations, `Source/engine/assets.cpp:` then a
+# list, and a location that leaves the file out, `(:507)`, which reads in
+# the file named last: devilutionx, corsixth and starship cite their load
+# order this way. Neither was collected, so no bump or guard followed them.
+_PROSE_FILE_NAMED_RE = re.compile(_PROSE_PATH + r":(?=\s|$)")
+_PROSE_ABBREVIATED_RE = re.compile(
+    r"(?:(?<=[(\s])|^):(?P<range>\d+(?:-\d+)?)(?![\w:-])(?!\.\d)"
 )
 PROSE_LIST_ITEM_RE = re.compile(r"\d+[.)](?:\s|$)")
 PROSE_WRAPPED_RANGE_RE = re.compile(r":\d+(?:-\d+)?(?:[,)\s]|$)")
@@ -344,6 +355,12 @@ def _prose_runs(
     joined line, so a writer can put each span back on its own line.
     """
     lines = text.splitlines()
+    # The file an abbreviated location reads in, and the role it was named
+    # under, carried from line to line within the scalar.
+    named: tuple[str, str] | None = None
+    # Where the run wrapped from the line before ends in this one: its
+    # ranges are that run's, not abbreviated locations of their own.
+    carried = 0
     for number, own in enumerate(lines):
         # A literal block keeps its line breaks, so a citation wrapped after
         # `file:` or after a range's comma lost the ranges on the next line
@@ -369,46 +386,35 @@ def _prose_runs(
         elif following and line.endswith(","):
             joined_at = len(line) + 1
             line += " " + following
-        consumed = 0
-        for match in PROSE_CITE_RE.finditer(line):
+        consumed, carried = carried, 0
+        events = sorted(
+            [(m.start(), 0, m) for m in PROSE_CITE_RE.finditer(line)]
+            + [(m.start(), 1, m) for m in _PROSE_FILE_NAMED_RE.finditer(line)]
+            + [(m.start(), 2, m) for m in _PROSE_ABBREVIATED_RE.finditer(line)],
+            key=lambda event: (event[0], event[1]),
+        )
+        for _start, kind, match in events:
             if match.start() >= boundary:
                 break
             if match.start() < consumed:
                 continue
-            token_start = max(line.rfind(" ", 0, match.start()),
-                              line.rfind("\t", 0, match.start())) + 1
-            token = line[token_start:].split()[0] if line[token_start:] else ""
-            if "://" in token:
+            if kind == 1:
+                if "://" not in line[max(0, match.start() - 3):match.end()]:
+                    named = (match.group("path"), "")
                 continue
-            path = match.group("path")
-            path_start = match.start("path")
-            # A dotfile citation opens its token with the dot: `.gitlab-ci.yml`
-            # after a delimiter is the filename, while the dot of `boot.cfg`
-            # inside a word belongs to the word before it.
-            if (
-                path_start > 0
-                and line[path_start - 1] == "."
-                and (path_start == 1 or line[path_start - 2] in " \t(")
-            ):
-                path_start -= 1
-                path = "." + path
-            # A path opening its line reads its role or project word at the
-            # end of the line before: genesis_plus_gx_expanded wrapped
-            # `(Batocera` and `package/batocera/...mk:7` one line apart.
-            lead = line[:path_start]
-            if number and not lead.strip():
-                lead = lines[number - 1].rstrip() + " "
-            role = _prose_role(lead, len(lead))
-            if not role and _prose_external(lead, len(lead), known):
-                # Kept with its project word, the source_ref convention: the
-                # report looks for the project inside the declared trees
-                # (mupen64plus-ae vendors `rice Config.cpp`) and calls the
-                # citation EXTERNAL only when none carries it.
-                before = lead.rstrip()
-                word = before[max(before.rfind(" "), before.rfind("\t"),
-                                  before.rfind("(")) + 1:]
-                path = f"{word} {path}"
-            path_span = (path_start, match.end("path"))
+            if kind == 2:
+                if named is None:
+                    continue
+                path, role = named
+                path_start = match.start()
+                path_span = (path_start, path_start)
+            else:
+                path, role, path_start, path_span = _prose_path(
+                    line, lines, number, match, known
+                )
+                if path is None:
+                    continue
+                named = (path, role)
             spans = [(match.start("range"), match.end("range"))]
             end = match.end()
             while True:
@@ -429,6 +435,9 @@ def _prose_runs(
                 spans.append((cont.start("range"), cont.end("range")))
                 end = cont.end()
             consumed = end
+            if end > boundary:
+                indent = len(lines[number + 1]) - len(lines[number + 1].lstrip())
+                carried = indent + end - joined_at
             parts = []
             for lo, hi in spans:
                 rng = line[lo:hi]
@@ -438,3 +447,44 @@ def _prose_runs(
                 number, path_span, tuple(spans), tuple(parts),
                 line[path_start:end], joined_at if end > boundary else 0,
             )
+
+
+def _prose_path(
+    line: str, lines: list[str], number: int, match: re.Match,
+    known: frozenset[str] | set[str],
+) -> tuple[str | None, str, int, tuple[int, int]]:
+    """(path, role, start, span) of one written citation, None for a URL."""
+    token_start = max(line.rfind(" ", 0, match.start()),
+                      line.rfind("\t", 0, match.start())) + 1
+    token = line[token_start:].split()[0] if line[token_start:] else ""
+    if "://" in token:
+        return None, "", 0, (0, 0)
+    path = match.group("path")
+    path_start = match.start("path")
+    # A dotfile citation opens its token with the dot: `.gitlab-ci.yml`
+    # after a delimiter is the filename, while the dot of `boot.cfg`
+    # inside a word belongs to the word before it.
+    if (
+        path_start > 0
+        and line[path_start - 1] == "."
+        and (path_start == 1 or line[path_start - 2] in " \t(")
+    ):
+        path_start -= 1
+        path = "." + path
+    # A path opening its line reads its role or project word at the
+    # end of the line before: genesis_plus_gx_expanded wrapped
+    # `(Batocera` and `package/batocera/...mk:7` one line apart.
+    lead = line[:path_start]
+    if number and not lead.strip():
+        lead = lines[number - 1].rstrip() + " "
+    role = _prose_role(lead, len(lead))
+    if not role and _prose_external(lead, len(lead), known):
+        # Kept with its project word, the source_ref convention: the
+        # report looks for the project inside the declared trees
+        # (mupen64plus-ae vendors `rice Config.cpp`) and calls the
+        # citation EXTERNAL only when none carries it.
+        before = lead.rstrip()
+        word = before[max(before.rfind(" "), before.rfind("\t"),
+                          before.rfind("(")) + 1:]
+        path = f"{word} {path}"
+    return path, role, path_start, (path_start, match.end("path"))

scripts/validation.py

@@ -16,6 +16,7 @@ from common import (
     adler32_of,
     archive_members,
     compute_hashes,
+    entry_system_ids,
     name_match_size_ok,
     sanitize_pack_path,
     size_fits,
@@ -145,6 +146,72 @@ def _rule_failure(
     return None
 
 
+def _found_by_content(entry: dict, profile: dict) -> bool:
+    """Whether every build of the core identifies this entry by content."""
+    return not opens_by_name(entry, profile) and not opens_by_name(
+        entry, profile, standalone=True
+    )
+
+
+def _regions(entry: dict) -> set[str]:
+    value = entry.get("region")
+    return set(value) if isinstance(value, list) else {value} if value else set()
+
+
+def content_alternatives(entry: dict, profile: dict) -> list[dict]:
+    """The labels a content-found core takes in place of this one.
+
+    rustation scans its folder by sha256 (found_by: content): its
+    scph1001.bin is a label for v2.2, and the v2.0 dump ROCKNIX ships under
+    that name boots the same North American console, labelled
+    scph1001_v20.bin. Alternatives share the entry's system and declared
+    region, the slot slot.py groups by: openMSX also finds its ROMs by
+    sha1, but each one serves its own machine and declares no region.
+    """
+    if not _found_by_content(entry, profile):
+        return []
+    regions = _regions(entry)
+    if not regions:
+        return []
+    systems = set(entry_system_ids(entry, profile))
+    return [
+        other for other in profile.get("files") or []
+        if isinstance(other, dict)
+        and _found_by_content(other, profile)
+        and regions & _regions(other)
+        and systems == set(entry_system_ids(other, profile))
+    ]
+
+
+def _labels_widened(profile: dict) -> list[dict]:
+    """The profile's entries, each content-found label accepting the dumps
+    of its alternatives (content_alternatives)."""
+    widened = []
+    for f in profile.get("files") or []:
+        if not isinstance(f, dict):
+            continue
+        siblings = content_alternatives(f, profile)
+        if len(siblings) < 2:
+            widened.append(f)
+            continue
+        merged = dict(f)
+        for field_name in ("crc32", "md5", "sha1", "sha256"):
+            if f.get(field_name):
+                merged[field_name] = list(dict.fromkeys(
+                    value for other in siblings
+                    for value in parse_hash_list(other.get(field_name))
+                ))
+        if f.get("size") is not None:
+            merged["size"] = sorted({
+                size for other in siblings
+                for size in (other["size"] if isinstance(other.get("size"), list)
+                             else [other.get("size")])
+                if size is not None
+            })
+        widened.append(merged)
+    return widened
+
+
 def _build_validation_index(profiles: dict) -> dict[str, dict]:
     """Build per-filename validation rules from emulator profiles.
 
@@ -170,7 +237,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
     for emu_name, profile in profiles.items():
         if profile.get("type") in ("launcher", "alias"):
             continue
-        for f in profile.get("files", []):
+        for f in _labels_widened(profile):
             fname = f.get("name", "")
             if not fname:
                 continue

scripts/verify.py

@@ -1845,6 +1845,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
 
     game_data, bios_files, unplaced = _split_undeclared(undeclared)
 
+    # A file installed with its emulator is never collected, firmware or
+    # game data, required or not: cross_reference counts it apart. The
+    # firmware already left the pack lists (_mark_unplaced), the game data
+    # did not, and Abuse's three bundled data files read as missing.
+    bundled = [u for u in undeclared if not u["in_repo"] and u.get("bundled")]
+    game_data = [u for u in game_data if u["in_repo"] or not u.get("bundled")]
+
     req_not_in_repo = _core_required_missing(bios_files)
     req_hle_not_in_repo = [
         u
@@ -1854,12 +1861,6 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
     req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]]
     opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]]
     opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]]
-
-    bundled = [
-        u for u in bios_files
-        if u["required"] and not u["in_repo"] and u.get("bundled")
-        and not u.get("hle_fallback")
-    ]
     # The builder's count, one file per pack path, when the platform pass
     # replayed it. Counting entries said 1130 for RetroArch, 48 more than the
     # pack holds: rom.key alone was claimed by three Amiga cores.
@@ -1886,8 +1887,17 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
         for u in opt_not_in_repo:
             _print_undeclared_entry(u, "MISSING (optional)", verbose, repeated)
     if bundled:
-        names = ", ".join(sorted({u["name"] for u in bundled}))
-        print(f"  Bundled with their emulator, not collected: {names}")
+        # The required firmware is named, as a missing one would be. The
+        # rest (shaders, runtime libraries, engine data) on request.
+        shown = bundled if verbose else [
+            u for u in bundled
+            if u["required"] and u.get("category", "bios") != "game_data"
+        ]
+        names = ", ".join(sorted({u["name"] for u in shown}))
+        print(
+            f"  Bundled with their emulator, not collected: {len(bundled)}"
+            + (f" ({names})" if names else "")
+        )
     if unplaced:
         print(f"  Core files held but not packed: {len(unplaced)} "
               f"({_omission_reasons(unplaced)})")