chore: merge main into the Starlight branch

scripts/dedup.py

@@ -70,6 +70,9 @@ NODEDUP_DIRS = {
     # MAME's support trees (hash, plugins) are read by path as a whole, and
     # two MAME versions share most of their software lists byte for byte.
     "mame",
+    # The Tyrian 2.1 distribution ships netset.pcx and tyrset.pcx with the
+    # same bytes, and EmuELEC places both names under ports/opentyrian/.
+    "opentyrian",
 }
 
 
@@ -311,7 +314,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
             if isinstance(previous, dict):
                 merged = dict(previous)
                 merged.update(mame_clones)
-                mame_clones = merged
+                mame_clones = collapse_clone_chains(merged)
         if dry_run:
             print(f"\nWould write MAME clone map: {clone_path}")
             print(
@@ -329,6 +332,43 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
     return results
 
 
+def collapse_clone_chains(clone_map: dict[str, dict]) -> dict[str, dict]:
+    """Point every clone at a canonical archive that still exists.
+
+    The map is cumulative. When a later run finds that a canonical archive is
+    itself a byte-identical copy of another and removes it, the entry that
+    named it as canonical would send its clones to a file that is gone:
+    sys68k_cpu21a pointed at sys68k_cpu21 after sys68k_cpu21 became a clone
+    of sys68k_cpu20. Each such entry's clones move to the final canonical.
+    """
+    owner = {
+        clone: canonical
+        for canonical, entry in clone_map.items()
+        for clone in entry.get("clones", [])
+    }
+
+    def final(name: str) -> str:
+        seen = {name}
+        while name in owner:
+            name = owner[name]
+            if name in seen:
+                raise ValueError(f"clone map loops through {name}")
+            seen.add(name)
+        return name
+
+    collapsed: dict[str, dict] = {}
+    for canonical, entry in clone_map.items():
+        target = final(canonical)
+        kept = collapsed.setdefault(target, {**clone_map.get(target, entry), "clones": []})
+        names = set(kept["clones"]) | set(entry.get("clones", []))
+        if target != canonical:
+            names.add(canonical)
+        names.discard(target)
+        kept["clones"] = sorted(names)
+        kept["total_copies"] = len(kept["clones"])
+    return collapsed
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(allow_abbrev=False, description="Deduplicate bios/ directory")
     parser.add_argument(

scripts/validate_schemas.py

@@ -8,6 +8,7 @@ import json
 import re
 import sys
 import zipfile
+from collections.abc import Callable
 from pathlib import Path, PurePosixPath
 
 import yaml
@@ -123,7 +124,11 @@ def _checks_without_values(path: Path, document: object) -> list[str]:
     return out
 
 
-def _validate_json_files(paths: list[Path], schema_name: str) -> list[str]:
+def _validate_json_files(
+    paths: list[Path],
+    schema_name: str,
+    semantic_check: Callable[[Path, object], list[str]] | None = None,
+) -> list[str]:
     validator = _validator(schema_name)
     out: list[str] = []
     for path in paths:
@@ -132,7 +137,10 @@ def _validate_json_files(paths: list[Path], schema_name: str) -> list[str]:
         except (OSError, json.JSONDecodeError) as exc:
             out.append(f"{path.relative_to(ROOT)}: {exc}")
             continue
-        out.extend(_errors(validator, data, str(path.relative_to(ROOT))))
+        validation_errors = _errors(validator, data, str(path.relative_to(ROOT)))
+        out.extend(validation_errors)
+        if not validation_errors and semantic_check is not None:
+            out.extend(semantic_check(path, data))
     return out
 
 
@@ -407,16 +415,20 @@ def main() -> int:
 
     if not args.source_only:
         database_path = ROOT / "database.json"
-        errors.extend(_validate_json_files([database_path], "database.schema.json"))
-        if database_path.exists():
-            errors.extend(_semantic_database_checks(_load_json(database_path)))
+        errors.extend(
+            _validate_json_files(
+                [database_path],
+                "database.schema.json",
+                lambda _path, database: _semantic_database_checks(database),
+            )
+        )
 
         install_paths = sorted((ROOT / "install").glob("*.json"))
         errors.extend(
-            _validate_json_files(install_paths, "install-manifest.schema.json")
+            _validate_json_files(
+                install_paths, "install-manifest.schema.json", _semantic_install_checks
+            )
         )
-        for path in install_paths:
-            errors.extend(_semantic_install_checks(path, _load_json(path)))
 
         target_paths = sorted((ROOT / "install" / "targets").glob("*.json"))
         errors.extend(
@@ -448,10 +460,12 @@ def main() -> int:
             (api_dir / "systems").glob("*.json")
         ) + sorted((api_dir / "lookup").glob("*.json"))
         errors.extend(
-            _validate_json_files(envelope_paths, "site-api-envelope.schema.json")
+            _validate_json_files(
+                envelope_paths,
+                "site-api-envelope.schema.json",
+                _semantic_envelope_checks,
+            )
         )
-        for path in envelope_paths:
-            errors.extend(_semantic_envelope_checks(path, _load_json(path)))
 
         errors.extend(_validate_pack_manifests(ROOT / "dist"))