chore: merge round 2 fixes (tests)

# Conflicts:
#	install.ps1
#	install.py
#	install.sh

scripts/check_release_assets.py

@@ -22,6 +22,7 @@ import re
 import subprocess
 import sys
 from collections.abc import Iterable
+from typing import NamedTuple
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import composition_tier, load_database
@@ -32,7 +33,14 @@ from largefiles import (
     registered_paths,
 )
 
-Finding = tuple[str, str, int, int | None]
+
+class AssetFinding(NamedTuple):
+    """A difference between the release and the collection."""
+
+    kind: str
+    path: str
+    size: int
+    published: int | None
 
 
 def expected_assets(db: dict, gitignore_text: str) -> dict[str, int]:
@@ -61,7 +69,7 @@ def compare(
     registered: Iterable[str] | None = None,
     expected_digests: dict[str, str] | None = None,
     asset_digests: dict[str, str] | None = None,
-) -> list[Finding]:
+) -> list[AssetFinding]:
     """Files the release lacks or serves at another size, sorted by kind.
 
     At the same size, the sha256 the release lists for an asset is held
@@ -74,9 +82,9 @@ def compare(
     matches what render_notes() would write is a finding of its own: the
     page is what a reader checks a download against.
     """
-    findings: list[Finding] = []
+    findings: list[AssetFinding] = []
     if not notes_current:
-        findings.append(("notes", "release description", 0, None))
+        findings.append(AssetFinding("notes", "release description", 0, None))
     names = asset_names([*(registered or ()), *expected])
     for path, size in sorted(expected.items()):
         served = next(
@@ -84,14 +92,14 @@ def compare(
         )
         published = assets[served] if served else None
         if published is None:
-            findings.append(("missing", path, size, None))
+            findings.append(AssetFinding("missing", path, size, None))
         elif published != size:
-            findings.append(("size", path, size, published))
+            findings.append(AssetFinding("size", path, size, published))
         else:
             ours = (expected_digests or {}).get(path)
             theirs = (asset_digests or {}).get(served)
             if ours and theirs and ours != theirs:
-                findings.append(("content", path, size, published))
+                findings.append(AssetFinding("content", path, size, published))
     return sorted(findings)
 
 

scripts/common.py

@@ -263,7 +263,6 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
 
     config, chain = _inherited_yaml(platform_name, platforms_dir)
 
-    # Resolve shared group includes
     shared_path = os.path.join(platforms_dir, "_shared.yml")
     # Merge metadata from _registry.yml. The registry is our curated source,
     # while the scraped YAML may be incomplete (missing cores, metadata fields).

scripts/cross_reference.py

@@ -706,7 +706,6 @@ def _cross_reference_profile(
 
         seen_files.add(seen_key)
 
-        # Group archived files by archive name
         if archive:
             archive_key = (
                 archive,
@@ -771,7 +770,6 @@ def _cross_reference_profile(
         }
         gaps.append(entry)
 
-    # Append grouped archive gaps
     for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
         gaps.append(ag)
 

scripts/crypto_verify.py

@@ -250,7 +250,6 @@ def verify_secure_info_a(
     if serial == b"\x00" * 15:
         return False, "invalid: serial_number is all zeros"
 
-    # Get RSA keys
     rsa_keys = keys.get("RSA", {})
     modulus = rsa_keys.get("secureInfoMod")
     exponent = rsa_keys.get("secureInfoExp")

scripts/dedup.py

@@ -301,7 +301,6 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
     if not dry_run and empty_cleaned:
         print(f"Cleaned {empty_cleaned} empty directories")
 
-    # Write MAME clone mapping
     if mame_clones:
         # A group is only visible while both copies are on disk, and this run
         # has just deleted the clone. Writing only what was seen this time

scripts/download.sh

@@ -66,7 +66,19 @@ normalize() {
     printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9'
 }
 
+# The anonymous quota is 60 calls an hour per address. The token goes to
+# GitHub's API only: RETROBIOS_API can name any host. The header is read
+# from stdin so the token stays out of the process list.
 fetch_release() {
+    case "$API_BASE" in
+        https://api.github.com|https://api.github.com/)
+            if [ -n "${GITHUB_TOKEN:-}" ]; then
+                printf 'Authorization: Bearer %s\n' "$GITHUB_TOKEN" |
+                    curl -fsSL -H @- "$API"
+                return
+            fi
+            ;;
+    esac
     curl -fsSL "$API"
 }
 

scripts/generate_pack.py

@@ -993,7 +993,6 @@ def generate_pack(
                             agnostic_resolved = True
 
                     if agnostic_resolved and local_path:
-                        # Write rename README
                         original_name = os.path.basename(local_path)
                         dest_name = file_entry.get("name", "")
                         if original_name != dest_name and agnostic_path:
@@ -1537,7 +1536,6 @@ def generate_emulator_pack(
             if not files and not profile.get("data_directories"):
                 print(f"  No files needed for {profile.get('emulator', emu_name)}")
 
-            # Collect archives as atomic units
             archives: set[str] = set()
             for fe in files:
                 archive = fe.get("archive")
@@ -4103,7 +4101,6 @@ def generate_manifest(
 
     repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-    # Filter systems by target
     plat_cores = _platform_cores(config, emu_profiles) if target_cores else None
     pack_systems = filter_systems_by_target(
         config.get("systems", {}),

scripts/generate_readme.py

@@ -711,7 +711,9 @@ def contributing_text(
 ## Add a BIOS file
 
 1. Fork this repository
-2. Place the file in `bios/Manufacturer/Console/filename`
+2. Place the file in `bios/Manufacturer/Console/filename`.
+   `python scripts/validate_pr.py path/to/file` names the folder its system
+   is kept in, and whether a platform or an emulator profile declares it
 3. Variants (alternate hashes for the same file): `bios/Manufacturer/Console/.variants/`
 4. Open a pull request. Its hashes are checked automatically and reported in a comment
 

scripts/scraper/_hash_merge.py

@@ -443,7 +443,6 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
     """
     import re
 
-    # Build a lookup of what to patch
     patches: dict[str, dict] = {}
     for fe in files:
         if fe.get("category") != "bios_zip":

scripts/scraper/batocera_scraper.py

@@ -444,7 +444,6 @@ class Scraper(BaseScraper):
         """Generate a platform YAML config dict from scraped data."""
         requirements = self.fetch_requirements()
 
-        # Parse source to extract display names per system
         raw = self._fetch_raw()
         source_dict = self._extract_systems_dict(raw)
         display_names: dict[str, str] = {}

scripts/scraper/bizhawk_scraper.py

@@ -214,7 +214,6 @@ def parse_firmware_database(
         r"\s*\)"
     )
 
-    # Collect firmware slots
     firmware_slots: dict[tuple[str, str], str] = {}
     for m in firmware_re.finditer(source):
         system, fw_id, desc = m.group(1), m.group(2), m.group(3)
@@ -252,7 +251,6 @@ def parse_firmware_database(
         slot_key = (system, fw_id)
         slot_options.setdefault(slot_key, []).append((file_entry, status))
 
-    # Build records from FirmwareAndOption one-liners
     records: list[dict] = []
 
     for m in fao_re.finditer(source):

scripts/scraper/mame_parser.py

@@ -372,13 +372,12 @@ def _parse_rom_entries(block: str) -> list[dict]:
         ("rom_load", _ROM_LOAD),
     ]
 
-    # Collect all matches with their positions
     events: list[tuple[int, str, re.Match]] = []
     for tag, pat in token_patterns:
         for m in pat.finditer(block):
             events.append((m.start(), tag, m))
 
-    # Sort by position in block
+    # Walked in source order: a ROM_LOAD belongs to the last ROM_REGION above it.
     events.sort(key=lambda e: e[0])
 
     for _pos, tag, m in events:

scripts/scraper/retrobat_scraper.py

@@ -230,7 +230,6 @@ class Scraper(BaseScraper):
         """Generate a platform YAML config dict from scraped data."""
         requirements = self.fetch_requirements()
 
-        # Parse source to extract display names per system
         raw = self._fetch_raw()
         source_data = json.loads(raw)
         display_names: dict[str, str] = {}

scripts/scraper/retrodeck_scraper.py

@@ -380,7 +380,6 @@ class Scraper(BaseScraper):
                     if system is None:
                         continue
 
-                    # Resolve path
                     paths_raw = entry.get("paths")
                     if isinstance(paths_raw, str):
                         resolved = _resolve_path(paths_raw)

scripts/sect233r1.py

@@ -212,11 +212,9 @@ def ecdsa_verify_sha256(
     if len(public_key_xy) != 60:
         return False
 
-    # Parse signature
     r = int.from_bytes(signature_rs[:30], "big")
     s = int.from_bytes(signature_rs[30:], "big")
 
-    # Parse public key
     qx = int.from_bytes(public_key_xy[:30], "big")
     qy = int.from_bytes(public_key_xy[30:], "big")
     q_point = (qx, qy)
@@ -225,21 +223,20 @@ def ecdsa_verify_sha256(
     if not (1 <= r < _N and 1 <= s < _N):
         return False
 
-    # Compute hash
     h = hashlib.sha256(message).digest()
     e = int.from_bytes(h, "big")
     # Truncate to bit length of n
     if 256 > _N_BITLEN:
         e >>= 256 - _N_BITLEN
 
-    # Compute w = s^(-1) mod n
+    # w = s^(-1) mod n
     w = _modinv(s, _N)
 
-    # Compute u1 = e*w mod n, u2 = r*w mod n
+    # u1 = e*w mod n, u2 = r*w mod n
     u1 = (e * w) % _N
     u2 = (r * w) % _N
 
-    # Compute R = u1*G + u2*Q
+    # R = u1*G + u2*Q
     g_point = (_Gx, _Gy)
     r_point = _ec_add(_ec_mul(u1, g_point), _ec_mul(u2, q_point))
 

scripts/validate_pr.py

@@ -20,6 +20,7 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
 import subprocess
 import sys
 
@@ -170,18 +171,37 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
                     _note_sizes(known["sizes"], [file_entry["name"]], file_entry)
                 for value in (*sha1s, *md5s):
                     systems_of.setdefault(value, set()).add(system_id)
-                held = next((s for s in sha1s if s in files), None) or next(
-                    (by_md5[m] for m in md5s if m in by_md5), None
-                )
-                parts = str(files.get(held, {}).get("path", "")).split("/")
-                if len(parts) >= 4 and parts[0] == "bios" and not parts[2].startswith("."):
-                    folder = "/".join(parts[:3]) + "/"
-                    counts = folders_of.setdefault(system_id, {})
-                    counts[folder] = counts.get(folder, 0) + 1
+                _note_folder(folders_of, system_id, files, by_md5, sha1s, md5s)
 
-    # A system whose files sit under several manufacturers (arcade sets,
-    # computers sharing ROM chips) names no single home: only a system
-    # filed under one manufacturer says where a newcomer belongs.
+    known["folders"] = _system_homes(systems_of, folders_of)
+    return known
+
+
+def _note_folder(
+    folders_of: dict[str, dict[str, int]], system_id: str, files: dict,
+    by_md5: dict, sha1s: list[str], md5s: list[str],
+) -> None:
+    """Count the bios/<Manufacturer>/<Console>/ folder a held file sits in."""
+    held = next((s for s in sha1s if s in files), None) or next(
+        (by_md5[m] for m in md5s if m in by_md5), None
+    )
+    parts = str(files.get(held, {}).get("path", "")).split("/")
+    if len(parts) >= 4 and parts[0] == "bios" and not parts[2].startswith("."):
+        folder = "/".join(parts[:3]) + "/"
+        counts = folders_of.setdefault(system_id, {})
+        counts[folder] = counts.get(folder, 0) + 1
+
+
+def _system_homes(
+    systems_of: dict[str, set[str]], folders_of: dict[str, dict[str, int]]
+) -> dict[str, set[str]]:
+    """Hash value to the folders the collection files its systems under.
+
+    A system whose files sit under several manufacturers (arcade sets,
+    computers sharing ROM chips) names no single home: only a system filed
+    under one manufacturer says where a newcomer belongs.
+    """
+    homes: dict[str, set[str]] = {}
     for value, systems in systems_of.items():
         folders: set[str] = set()
         for system_id in systems:
@@ -189,19 +209,33 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
             if len({folder.split("/")[1] for folder in held_in}) == 1:
                 folders.update(held_in)
         if folders:
-            known["folders"][value] = folders
-    return known
+            homes[value] = folders
+    return homes
 
 
-def load_profile_hashes(emulators_dir: str) -> dict[str, dict[str, set[str]]]:
+def load_profile_hashes(
+    emulators_dir: str, db: dict | None = None
+) -> dict[str, dict[str, set[str]]]:
     """Hash value to the emulators whose profile declares it, per algorithm.
 
     A file only a core reads (a standalone emulator's firmware, a game
     engine's data) is in no platform list: the review called every such
-    contribution "not referenced, needs manual review".
+    contribution "not referenced, needs manual review". With the database,
+    `folders` maps each declared hash to the folders the collection files
+    the entry's system under, as load_platform_hashes does: a contributor
+    holding a file only a profile declares was told to pick a
+    bios/Manufacturer/Console/ folder alone.
     """
-    known: dict[str, dict] = {"sha1": {}, "sha256": {}, "md5": {}, "crc32": {}, "sizes": {}}
+    known: dict[str, dict] = {
+        "sha1": {}, "sha256": {}, "md5": {}, "crc32": {}, "sizes": {}, "folders": {},
+    }
+    systems_of: dict[str, set[str]] = {}
+    folders_of: dict[str, dict[str, int]] = {}
+    files = (db or {}).get("files", {})
+    indexes = (db or {}).get("indexes", {})
+    by_md5, by_sha256 = indexes.get("by_md5", {}), indexes.get("by_sha256", {})
     for emulator, profile in load_emulator_profiles(emulators_dir).items():
+        declared_systems = profile.get("systems") or []
         for entry in profile.get("files") or []:
             if not isinstance(entry, dict):
                 continue
@@ -211,9 +245,23 @@ def load_profile_hashes(emulators_dir: str) -> dict[str, dict[str, set[str]]]:
             # sha256), and a crc32 counts only where no stronger hash is
             # declared, bounded by the declared size (626 entries name a
             # file by crc32 alone, clk's Electron os.rom among them).
-            for algo in ("sha1", "sha256", "md5"):
-                for value in parse_hash_list(entry.get(algo)):
+            hashes = {
+                algo: parse_hash_list(entry.get(algo)) for algo in ("sha1", "sha256", "md5")
+            }
+            for algo, values in hashes.items():
+                for value in values:
                     known[algo].setdefault(value, set()).add(emulator)
+            # The entry's own system, or the profile's when it emulates one.
+            system = entry.get("system") or (
+                declared_systems[0] if len(declared_systems) == 1 else None
+            )
+            if isinstance(system, str):
+                for value in (v for values in hashes.values() for v in values):
+                    systems_of.setdefault(value, set()).add(system)
+                sha1s = [*hashes["sha1"], *(
+                    by_sha256[v] for v in hashes["sha256"] if v in by_sha256
+                )]
+                _note_folder(folders_of, system, files, by_md5, sha1s, hashes["md5"])
             if not any(entry.get(algo) for algo in ("sha1", "sha256", "md5")):
                 size = entry.get("size")
                 sizes = [int(v) for v in size] if isinstance(size, list) else (
@@ -223,6 +271,7 @@ def load_profile_hashes(emulators_dir: str) -> dict[str, dict[str, set[str]]]:
                     for accepted in sizes:
                         key = (value.zfill(8), accepted)
                         known["crc32"].setdefault(key, set()).add(emulator)
+    known["folders"] = _system_homes(systems_of, folders_of)
     return known
 
 
@@ -334,14 +383,20 @@ def validate_file(
         )
 
     normalized = os.path.normpath(relative)
-    expected_folders = sorted(
-        platform_hashes.get("folders", {}).get(result.sha1)
-        or platform_hashes.get("folders", {}).get(result.md5) or set()
+    # A platform's declaration first, then a profile's: most files the gaps
+    # page lists are declared by a profile alone.
+    expected = next(
+        (
+            table["folders"][value]
+            for table in (platform_hashes, profile_hashes)
+            for value in (result.sha1, result.md5, hashes.get("sha256", ""))
+            if value in table.get("folders", {})
+        ),
+        set(),
     )
+    expected_folders = sorted(expected)
     if normalized.startswith("bios" + os.sep):
         parts = normalized.split(os.sep)
-        expected = (platform_hashes.get("folders", {}).get(result.sha1)
-                    or platform_hashes.get("folders", {}).get(result.md5) or set())
         here = f"bios/{parts[1]}/{parts[2]}/" if len(parts) >= 4 else ""
         makers = {folder.split("/")[1] for folder in expected}
         if expected and (len(parts) < 4 or parts[1] not in makers):
@@ -413,20 +468,67 @@ def _git_lines(*args: str) -> list[str] | None:
     return [line for line in result.stdout.splitlines() if line]
 
 
+PROJECT_URLS = (
+    "https://github.com/Abdess/retrobios",
+    "https://github.com/Abdess/retroarch_system",
+)
+
+
+def _repo_key(url: str) -> str:
+    """owner/name of a clone URL, whatever its scheme."""
+    path = url.strip().rstrip("/").removesuffix(".git")
+    return "/".join(re.split(r"[/:]", path)[-2:]).casefold()
+
+
+def _project_main_line() -> str | None:
+    """The project's main line, fetched when no remote of this clone names it.
+
+    A fork's clone has origin alone. Once the commit is pushed to the fork's
+    main, origin/main is HEAD and the diff against it was empty, so --changed
+    found nothing and exited 0. None leaves the upstream/ and origin/ lines
+    in charge: a remote of that name names the project, or the clone has no
+    remote at all and only its uncommitted files can be read.
+    """
+    remotes = _git_lines("remote") or []
+    if not remotes:
+        return None
+    project = {_repo_key(url) for url in PROJECT_URLS}
+    for remote in remotes:
+        urls = _git_lines("remote", "get-url", "--all", remote) or []
+        if any(_repo_key(url) in project for url in urls):
+            return None if remote in ("upstream", "origin") else f"{remote}/main"
+    fetched = subprocess.run(
+        ["git", "fetch", "--quiet", "--no-tags", PROJECT_URLS[0], "main"],
+        capture_output=True, text=True, check=False,
+    )
+    if fetched.returncode != 0:
+        raise RuntimeError(
+            f"no remote of this clone is {PROJECT_URLS[0]}, and fetching its "
+            f"main failed: {fetched.stderr.strip()}"
+        )
+    return "FETCH_HEAD"
+
+
 def get_changed_files() -> list[str]:
     """BIOS files this checkout changes against the project's main line.
 
-    Committed changes are read against upstream/<base> first, then
-    origin/<base>: on a fork whose main carries the commit, origin/main is
-    HEAD and the diff was empty. Uncommitted and untracked files count too,
+    Committed changes are read against the project's main line: the remote
+    that names the project (upstream/<base> first, then origin/<base>), or
+    the project's main fetched when no remote does. Uncommitted and
+    untracked files count too,
     since CONTRIBUTING has the file copied into bios/ before any commit.
     A git that fails outright is an error, never "nothing changed".
     """
     changed: list[str] = []
     compared = False
-    for remote in ("upstream", "origin"):
-        for base in ("main", "master", "v2"):
-            committed = _git_lines("diff", "--name-only", f"{remote}/{base}...HEAD")
+    project_base = _project_main_line()
+    remotes = [[project_base]] if project_base else [
+        [f"{remote}/{base}" for base in ("main", "master", "v2")]
+        for remote in ("upstream", "origin")
+    ]
+    for candidates in remotes:
+        for line in candidates:
+            committed = _git_lines("diff", "--name-only", f"{line}...HEAD")
             if committed is None:
                 continue
             compared = True
@@ -512,6 +614,16 @@ def main() -> None:
     output.add_argument("--json", action="store_true", help="Output as JSON")
     args = parser.parse_args()
 
+    # The database, platforms and profiles are read from the repository
+    # root. Run from elsewhere, every file read as referenced by no platform
+    # and placed outside bios/. The files named are read from where they
+    # were typed.
+    from verify import REPO_ROOT, work_from_repo_root
+
+    if os.path.realpath(os.getcwd()) != os.path.realpath(REPO_ROOT):
+        args.files = [os.path.abspath(f) for f in args.files]
+    work_from_repo_root(args, parser, ["db", "platforms_dir", "emulators_dir"])
+
     files = _files_to_validate(args, parser)
     if not files:
         if args.json:
@@ -520,7 +632,7 @@ def main() -> None:
 
     db = _load_database_if_any(args.db)
     platform_hashes = load_platform_hashes(args.platforms_dir, db)
-    profile_hashes = load_profile_hashes(args.emulators_dir)
+    profile_hashes = load_profile_hashes(args.emulators_dir, db)
 
     results = []
     for f in files: