chore: merge doubled destinations in exports

scripts/export_native.py

@@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex
 import yaml
 
 from common import (
+    build_zip_contents_index,
     platform_arg,
     list_registered_platforms,
     load_data_dir_registry,
@@ -50,6 +51,7 @@ from common import (
 from exporter import discover_exporters
 from exporter.baseline import HASH_FIELDS, _hash_values, _single_size, build_native_model
 from exporter.candidates import removal_candidates
+from exporter.doubled import doubled_destinations
 from truth import generate_platform_truth
 
 DEFAULT_CACHE = ".cache/upstream-native"
@@ -436,6 +438,24 @@ def refresh_cache(
     return True, [f"{len(wanted)} file(s) fetched"]
 
 
+@functools.cache
+def _collection(db_path: str) -> tuple[dict, dict]:
+    """The database and its archive index, read once for every platform."""
+    db = load_database(db_path)
+    return db, build_zip_contents_index(db)
+
+
+def doubled_in(scraped: dict | None, platforms_dir: str) -> list[dict]:
+    """The destinations the platform's own list declares for two files."""
+    if not scraped:
+        return []
+    db, zip_contents = _collection(TRUTH_DB)
+    return doubled_destinations(
+        scraped, load_emulator_profiles("emulators"), db, zip_contents,
+        load_data_dir_registry(platforms_dir),
+    )
+
+
 def _count(number: int, noun: str, plural: str = "") -> str:
     """"1 hash", "2 hashes": a count with its noun in the right number."""
     plural = plural or noun + ("es" if noun.endswith(("sh", "ch", "s", "x")) else "s")
@@ -764,6 +784,8 @@ def export_platform(
         systems, scraped, load_emulator_profiles("emulators"),
         load_data_dir_registry(platforms_dir),
     )
+    # Reported, never rewritten: both declarations stay as the platform wrote them.
+    doubled = doubled_in(scraped, platforms_dir)
     messages.append(summary)
     if unwritten:
         messages.append("not in the file: " + ", ".join(unwritten))
@@ -774,6 +796,11 @@ def export_platform(
             + (f" ({open_question} undetermined)" if open_question else "")
             + ", listed in the report"
         )
+    if doubled:
+        messages.append(
+            f"{_count(len(doubled), 'destination')} declared twice for two files"
+            ", listed in the report"
+        )
     messages.extend(f"withheld: {reason}" for reason in withheld)
     for heading, labels in (
         ("hash updated", applied),
@@ -807,6 +834,7 @@ def export_platform(
         "withheld": list(withheld),
         "found_by_content": report.labels,
         "removal_candidates": candidates,
+        "doubled": doubled,
         "invalid": issues,
     })
     return True, messages

scripts/exporter/doubled.py

@@ -0,0 +1,137 @@
+"""Destinations a platform's own list declares twice, for two files.
+
+One path holds one file, so on an install one of the two checks always
+fails. RetroDECK declares bios/disk.rom for the PC-88 core and for XRoar,
+with two md5 that name two different ROMs. slots.find_collisions finds
+these. The export keeps both entries and names, beside each declaration,
+the emulators whose profile reads the file it declares.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from common import (
+    contents_members,
+    holds_required_members,
+    platform_layouts,
+    resolve_platform_cores,
+    standalone_set,
+)
+from exporter.baseline import HASH_FIELDS, _hash_values
+from slots import _accepted_by_all, _lists_hashes, find_collisions
+from validation import opens_by_name, outside_gap_scope
+
+
+def _names(entry: dict) -> set[str]:
+    """The file names a profile entry is opened under."""
+    names = {entry.get("name"), *(entry.get("aliases") or [])}
+    for key in ("path", "standalone_path"):
+        if entry.get(key):
+            names.add(str(entry[key]).rsplit("/", 1)[-1])
+    return {str(n).casefold() for n in names if n}
+
+
+def _size_fits(entry: dict, local: str) -> bool:
+    """Whether the file's size is one the entry states, or none is stated."""
+    try:
+        size = os.path.getsize(local)
+    except OSError:
+        return False
+    declared = entry.get("size")
+    sizes = declared if isinstance(declared, list) else [declared]
+    sizes = [s for s in sizes if isinstance(s, int)]
+    if sizes and size not in sizes:
+        return False
+    if isinstance(entry.get("min_size"), int) and size < entry["min_size"]:
+        return False
+    return not (isinstance(entry.get("max_size"), int) and size > entry["max_size"])
+
+
+def _reads(entry: dict, by_name: bool, local: str, db: dict) -> bool:
+    """Whether a profile entry describes the file at local.
+
+    An archive is read by its members, a dump by its hashes. A name the code
+    opens with no hash to check is read by any copy of a size it accepts. A
+    name that is only a label (a core identifying files by content) reads
+    nothing by name.
+    """
+    if entry.get("contents"):
+        return holds_required_members(local, contents_members(entry))
+    if _lists_hashes(entry):
+        return _accepted_by_all(local, [entry], db)
+    return by_name and _size_fits(entry, local)
+
+
+def _hashes(entry: dict) -> dict[str, list[str]]:
+    found = {f: _hash_values(entry, f) for f in HASH_FIELDS}
+    return {f: values for f, values in found.items() if values}
+
+
+def doubled_destinations(
+    scraped: dict | None,
+    profiles: dict[str, dict],
+    db: dict,
+    zip_contents: dict | None = None,
+    data_registry: dict | None = None,
+) -> list[dict]:
+    """Each destination the platform declares twice for two files.
+
+    Every declaration is listed with its system, component, hashes and the
+    file it resolves to, and the emulators the platform runs whose profile
+    reads that file there, with the profile's source_ref. Nothing is
+    removed from the platform's file.
+    """
+    if not scraped:
+        return []
+    collisions = find_collisions(scraped, db, zip_contents, data_registry)
+    if not collisions:
+        return []
+    standalone = standalone_set(scraped)
+    named: dict[str, list[tuple[str, dict, dict, bool]]] = {}
+    for key in sorted(resolve_platform_cores(scraped, profiles)):
+        profile = profiles[key]
+        if profile.get("type") in ("launcher", "alias"):
+            continue
+        for layout, is_standalone in platform_layouts(key, profile, standalone):
+            for entry in layout.get("files") or []:
+                if not isinstance(entry, dict) or outside_gap_scope(entry, is_standalone):
+                    continue
+                by_name = opens_by_name(entry, layout, is_standalone)
+                for name in _names(entry):
+                    named.setdefault(name, []).append((key, layout, entry, by_name))
+
+    doubled = []
+    for collision in collisions:
+        base = collision.destination.rsplit("/", 1)[-1].casefold()
+        declarations = []
+        for entry, local in collision.declarations:
+            readers: list[dict] = []
+            for key, layout, profile_entry, by_name in named.get(base, []):
+                if not local or not _reads(profile_entry, by_name, local, db):
+                    continue
+                reader = {
+                    "emulator": layout.get("emulator") or key,
+                    "profile": key,
+                    "entry": profile_entry.get("name", ""),
+                    "source_ref": profile_entry.get("source_ref", ""),
+                }
+                if reader not in readers:
+                    readers.append(reader)
+            declarations.append({
+                "system": entry.get("native_system") or "",
+                "component": entry.get("component") or "",
+                "name": entry.get("name", ""),
+                "hashes": _hashes(entry),
+                "file": local,
+                "readers": readers,
+            })
+        doubled.append({
+            "destination": collision.destination,
+            "declarations": declarations,
+        })
+    return doubled

scripts/siteexports.py

@@ -156,6 +156,49 @@ def candidate_lines(candidates: list[dict]) -> list[str]:
     return lines
 
 
+def _declaration(declaration: dict) -> str:
+    """One declaration of a doubled destination: who declares it, its hash,
+    and the emulators that read the file it names."""
+    where = f"`{declaration['system']}`"
+    if declaration.get("component"):
+        where += f" (`{declaration['component']}`)"
+    parts = [where] + [
+        f"{field} {_value(values[0] if len(values) == 1 else values)}"
+        for field, values in (declaration.get("hashes") or {}).items()
+    ]
+    readers = declaration.get("readers") or []
+    read = (
+        "read by " + ", ".join(
+            r["emulator"] + (f" (`{r['source_ref']}`)" if r.get("source_ref") else "")
+            for r in readers
+        )
+        if readers else "no emulator this platform runs is known to read this file"
+    )
+    return " ".join(parts) + f": {read}"
+
+
+def doubled_lines(doubled: list[dict]) -> list[str]:
+    """The destinations the original declares twice, for two files.
+
+    One path holds one file, so one of the two checks fails on every
+    install. Both entries stay in the updated file.
+    """
+    if not doubled:
+        return []
+    count = _plural(len(doubled), "destination")
+    lines = [
+        f"{count} of the original {'is' if len(doubled) == 1 else 'are'} declared "
+        "twice, for two different files. One path holds one file, so one of the "
+        "two checks fails on every install. Both entries stay in the updated file.",
+        "",
+    ]
+    for item in doubled:
+        lines.append(f"- `{item['destination']}`")
+        lines.extend(f"  - {_declaration(d)}" for d in item["declarations"])
+    lines.append("")
+    return lines
+
+
 def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]:
     """Every platform report the export wrote, by platform name."""
     root = Path(directory)
@@ -266,6 +309,7 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
                 "",
             ])
         lines.extend(candidate_lines(report.get("removal_candidates") or []))
+        lines.extend(doubled_lines(report.get("doubled") or []))
         if report.get("invalid"):
             lines.extend(markup.aside(
                 "warning", "The updated file does not pass the platform's own checks",

scripts/slots.py

@@ -675,6 +675,8 @@ class Collision:
 
     destination: str
     resolved: list[str]
+    # Each declaration with the file it resolved to, in the platform's order.
+    declarations: list[tuple[dict, str | None]] = field(default_factory=list)
 
 
 def _collision_json(collision: Collision) -> dict:
@@ -714,6 +716,7 @@ def find_collisions(
         if len(entries) < 2:
             continue
         resolved = []
+        declarations = []
         for entry in entries:
             local, _ = resolve_local_file(
                 entry,
@@ -722,6 +725,7 @@ def find_collisions(
                 dest_hint=entry.get("destination", ""),
                 data_dir_registry=data_dir_registry,
             )
+            declarations.append((entry, local))
             if local and local not in resolved:
                 resolved.append(local)
         if len(resolved) > 1 and not _same_file_family(resolved):
@@ -730,7 +734,9 @@ def find_collisions(
             # also names the first one's file.
             if any(_accepted_by_all(path, entries, db) for path in resolved):
                 continue
-            collisions.append(Collision(destination=key, resolved=resolved))
+            collisions.append(
+                Collision(destination=key, resolved=resolved, declarations=declarations)
+            )
     return collisions