feat: audit profile refs against upstream

scripts/check_profile_refs.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+"""Check emulator profile source_refs against the profiled upstream.
+
+Resolves the upstream commit in effect at each profile's profiled_date
+(or uses source_commit when the profile carries one), fetches every file
+a source_ref points to at that commit and at HEAD, and reports whether
+the values the profile declares still sit where the ref says.
+
+A ref is checked by looking for the entry's declared hashes (or filename
+when no hash exists) in a window around the cited lines. Three outcomes
+per ref at each revision: anchored (found in the window), moved (found
+elsewhere in the file), gone (absent from the file).
+
+Usage:
+    python scripts/check_profile_refs.py --emulator vice
+    python scripts/check_profile_refs.py --all
+    python scripts/check_profile_refs.py --emulator clk --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+sys.path.insert(0, os.path.dirname(__file__))
+from common import load_emulator_profiles
+
+WINDOW = 40
+RAW_URL = "https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}"
+API_COMMITS = (
+    "https://api.github.com/repos/{owner}/{repo}/commits"
+    "?until={date}T23:59:59Z&per_page=1"
+)
+
+_GITHUB_RE = re.compile(r"https?://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$")
+_REF_RE = re.compile(r"^(?P<path>[^:]+?)(?::(?P<start>\d+)(?:-(?P<end>\d+))?)?$")
+
+_file_cache: dict[tuple[str, str, str, str], list[str] | None] = {}
+
+
+def parse_source_ref(ref: str) -> tuple[str, int | None, int | None]:
+    """Split 'path/file.cpp:123-129' into (path, start, end)."""
+    m = _REF_RE.match(ref.strip())
+    if not m:
+        return ref.strip(), None, None
+    start = int(m.group("start")) if m.group("start") else None
+    end = int(m.group("end")) if m.group("end") else start
+    return m.group("path"), start, end
+
+
+def collect_tokens(entry: dict) -> list[str]:
+    """Declared values worth searching for near the ref: hashes, then name."""
+    tokens: list[str] = []
+    for field in ("sha1", "md5", "crc32", "sha256", "known_hash_adler32"):
+        val = entry.get(field)
+        vals = val if isinstance(val, list) else [val] if val else []
+        for v in vals:
+            v = str(v).lower().removeprefix("0x")
+            if v:
+                tokens.append(v)
+    if not tokens:
+        name = entry.get("name", "")
+        if name:
+            tokens.append(os.path.basename(name).lower())
+    return tokens
+
+
+def match_ref(
+    lines: list[str] | None,
+    start: int | None,
+    end: int | None,
+    tokens: list[str],
+) -> str:
+    """Return anchored / moved / gone for one ref against one revision."""
+    if lines is None:
+        return "gone"
+    text = [ln.lower() for ln in lines]
+    if start is not None:
+        lo = max(0, start - 1 - WINDOW)
+        hi = min(len(text), (end or start) + WINDOW)
+        window = text[lo:hi]
+        if any(tok in ln for tok in tokens for ln in window):
+            return "anchored"
+    if any(tok in ln for tok in tokens for ln in text):
+        return "moved" if start is not None else "anchored"
+    return "gone"
+
+
+def _github_repo(url: str) -> tuple[str, str] | None:
+    m = _GITHUB_RE.match(url.strip())
+    return (m.group(1), m.group(2)) if m else None
+
+
+def _http_json(url: str) -> object:
+    headers = {"User-Agent": "retrobios-refcheck/1.0"}
+    token = os.environ.get("GITHUB_TOKEN", "")
+    if token:
+        headers["Authorization"] = f"token {token}"
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return json.loads(resp.read().decode())
+
+
+def resolve_commit_at(owner: str, repo: str, date: str) -> str | None:
+    """Last default-branch commit on or before the given date."""
+    url = API_COMMITS.format(owner=owner, repo=repo, date=date)
+    try:
+        data = _http_json(url)
+    except (urllib.error.URLError, urllib.error.HTTPError) as exc:
+        print(f"  {owner}/{repo}: commit lookup failed: {exc}", file=sys.stderr)
+        return None
+    if isinstance(data, list) and data:
+        return data[0].get("sha")
+    return None
+
+
+def fetch_lines(owner: str, repo: str, ref: str, path: str) -> list[str] | None:
+    key = (owner, repo, ref, path)
+    if key in _file_cache:
+        return _file_cache[key]
+    url = RAW_URL.format(
+        owner=owner, repo=repo, ref=ref, path=urllib.parse.quote(path)
+    )
+    headers = {"User-Agent": "retrobios-refcheck/1.0"}
+    try:
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            lines = resp.read().decode("utf-8", errors="replace").splitlines()
+    except urllib.error.HTTPError as exc:
+        if exc.code != 404:
+            print(f"  fetch {url}: HTTP {exc.code}", file=sys.stderr)
+        lines = None
+    except urllib.error.URLError as exc:
+        print(f"  fetch {url}: {exc}", file=sys.stderr)
+        lines = None
+    _file_cache[key] = lines
+    return lines
+
+
+def check_profile(name: str, profile: dict) -> dict:
+    """Check every source_ref of one profile. Returns a report dict."""
+    report: dict = {
+        "profile": name,
+        "repos": [],
+        "pinned_commit": None,
+        "commit_source": None,
+        "refs": [],
+        "skipped": None,
+    }
+
+    repos = []
+    for field in ("upstream", "source"):
+        url = profile.get(field, "")
+        gh = _github_repo(url) if url else None
+        if gh and gh not in repos:
+            repos.append(gh)
+    if not repos:
+        report["skipped"] = "no github repository in upstream/source"
+        return report
+    report["repos"] = [f"{o}/{r}" for o, r in repos]
+
+    pinned = profile.get("source_commit")
+    if pinned:
+        report["commit_source"] = "source_commit"
+    else:
+        date = str(profile.get("profiled_date", ""))
+        if not date:
+            report["skipped"] = "no source_commit and no profiled_date"
+            return report
+        for owner, repo in repos:
+            pinned = resolve_commit_at(owner, repo, date)
+            if pinned:
+                repos = [(owner, repo)] + [r for r in repos if r != (owner, repo)]
+                break
+        report["commit_source"] = f"resolved from profiled_date {date}"
+    if not pinned:
+        report["skipped"] = "commit resolution failed"
+        return report
+    report["pinned_commit"] = pinned
+
+    rank = {"gone": 0, "moved": 1, "anchored": 2}
+    for entry in profile.get("files", []):
+        ref = entry.get("source_ref", "")
+        if not ref:
+            continue
+        tokens = collect_tokens(entry)
+
+        # A source_ref may carry several comma-separated references;
+        # the entry holds as long as one of them does.
+        pin_status = "gone"
+        head_status = "gone"
+        for part in (p.strip() for p in ref.split(",") if p.strip()):
+            path, start, end = parse_source_ref(part)
+            for owner, repo in repos:
+                pin_lines = fetch_lines(owner, repo, pinned, path)
+                if pin_lines is None and (owner, repo) != repos[-1]:
+                    continue
+                head_lines = fetch_lines(owner, repo, "HEAD", path)
+                pin_part = match_ref(pin_lines, start, end, tokens)
+                head_part = match_ref(head_lines, start, end, tokens)
+                if rank[pin_part] > rank[pin_status]:
+                    pin_status = pin_part
+                if rank[head_part] > rank[head_status]:
+                    head_status = head_part
+                break
+
+        report["refs"].append({
+            "name": entry.get("name", ""),
+            "source_ref": ref,
+            "at_pin": pin_status,
+            "at_head": head_status,
+        })
+    return report
+
+
+def _print_report(report: dict) -> None:
+    name = report["profile"]
+    if report["skipped"]:
+        print(f"{name}: skipped ({report['skipped']})")
+        return
+    refs = report["refs"]
+    pin = report["pinned_commit"][:12]
+    print(f"{name}: {len(refs)} refs against {pin} ({report['commit_source']})")
+    for state, where in (("at_pin", "pin"), ("at_head", "HEAD")):
+        counts: dict[str, int] = {}
+        for r in refs:
+            counts[r[state]] = counts.get(r[state], 0) + 1
+        summary = ", ".join(f"{v} {k}" for k, v in sorted(counts.items()))
+        print(f"  at {where}: {summary}")
+    for r in refs:
+        if r["at_pin"] != "anchored" or r["at_head"] != "anchored":
+            print(
+                f"  {r['source_ref']} ({r['name']}): "
+                f"pin={r['at_pin']} HEAD={r['at_head']}"
+            )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Check profile source_refs against upstream revisions"
+    )
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--emulator", help="check a single profile")
+    group.add_argument("--all", action="store_true", help="check every profile")
+    parser.add_argument("--emulators-dir", default="emulators")
+    parser.add_argument("--json", action="store_true", dest="as_json")
+    args = parser.parse_args()
+
+    profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
+    if args.emulator:
+        if args.emulator not in profiles:
+            print(f"unknown profile: {args.emulator}", file=sys.stderr)
+            sys.exit(1)
+        selected = {args.emulator: profiles[args.emulator]}
+    else:
+        selected = {
+            k: v
+            for k, v in sorted(profiles.items())
+            if v.get("type") not in ("alias", "test")
+        }
+
+    reports = [check_profile(name, prof) for name, prof in selected.items()]
+    if args.as_json:
+        print(json.dumps(reports, indent=2))
+    else:
+        for report in reports:
+            _print_report(report)
+
+
+if __name__ == "__main__":
+    main()

tests/test_profile_refs.py

@@ -0,0 +1,83 @@
+"""Tests for check_profile_refs pure functions (no network)."""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
+
+from check_profile_refs import collect_tokens, match_ref, parse_source_ref
+
+
+class TestParseSourceRef(unittest.TestCase):
+    def test_path_with_single_line(self):
+        self.assertEqual(
+            parse_source_ref("src/core/bios.cpp:258"),
+            ("src/core/bios.cpp", 258, 258),
+        )
+
+    def test_path_with_line_range(self):
+        self.assertEqual(
+            parse_source_ref("Machines/Utility/ROMCatalogue.cpp:123-129"),
+            ("Machines/Utility/ROMCatalogue.cpp", 123, 129),
+        )
+
+    def test_path_without_line(self):
+        self.assertEqual(
+            parse_source_ref("FirmwareDatabase.cs"),
+            ("FirmwareDatabase.cs", None, None),
+        )
+
+
+class TestCollectTokens(unittest.TestCase):
+    def test_hashes_preferred_over_name(self):
+        entry = {
+            "name": "kernal.bin",
+            "sha1": "6C4FA9465F6091B174DF27DFE679499DF447503C",
+            "crc32": "789c8cc5",
+        }
+        tokens = collect_tokens(entry)
+        self.assertIn("6c4fa9465f6091b174df27dfe679499df447503c", tokens)
+        self.assertIn("789c8cc5", tokens)
+        self.assertNotIn("kernal.bin", tokens)
+
+    def test_name_fallback_without_hashes(self):
+        tokens = collect_tokens({"name": "GC/USA/IPL.bin"})
+        self.assertEqual(tokens, ["ipl.bin"])
+
+    def test_adler_prefix_stripped(self):
+        tokens = collect_tokens({"known_hash_adler32": "0x4f1f6f5c"})
+        self.assertEqual(tokens, ["4f1f6f5c"])
+
+    def test_hash_lists(self):
+        tokens = collect_tokens({"md5": ["AABB", "ccdd"]})
+        self.assertEqual(tokens, ["aabb", "ccdd"])
+
+
+class TestMatchRef(unittest.TestCase):
+    LINES = ["x = 0"] * 100 + ['hash = "789c8cc5"'] + ["y = 1"] * 100
+
+    def test_anchored_inside_window(self):
+        self.assertEqual(
+            match_ref(self.LINES, 101, 101, ["789c8cc5"]), "anchored"
+        )
+
+    def test_moved_outside_window(self):
+        self.assertEqual(match_ref(self.LINES, 5, 5, ["789c8cc5"]), "moved")
+
+    def test_gone_when_absent(self):
+        self.assertEqual(match_ref(self.LINES, 101, 101, ["deadbeef"]), "gone")
+
+    def test_gone_when_file_missing(self):
+        self.assertEqual(match_ref(None, 1, 1, ["789c8cc5"]), "gone")
+
+    def test_anchored_no_line_searches_whole_file(self):
+        self.assertEqual(
+            match_ref(self.LINES, None, None, ["789c8cc5"]), "anchored"
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/tools.md

@@ -170,6 +170,31 @@ python scripts/export_native.py --platform batocera
 python scripts/export_native.py --all --output-dir dist/upstream/
 ```
 
+MAME and FBNeo entries are `.zip` ROM sets: their meaningful hashes belong
+to the files inside the archive, so the exported DAT lists those entries
+without a container sha1. Anyone submitting the DAT upstream should mention
+this.
+
+### check_profile_refs.py
+
+Audit `source_ref` line references against the profiled upstream. The
+commit under audit is the profile's `source_commit` when present, else the
+last upstream commit at `profiled_date`. Each referenced file is fetched
+at that commit and at HEAD, and the entry's declared hashes (or filename)
+are searched around the cited lines.
+
+```bash
+python scripts/check_profile_refs.py --emulator vice
+python scripts/check_profile_refs.py --all --json
+```
+
+Per ref and revision: `anchored` (found at the cited lines), `moved`
+(found elsewhere in the file), `gone` (absent from the file). `moved` at
+HEAD means upstream shifted since profiling; `gone` at pin means the
+declared value does not come from the referenced file and the entry
+deserves a re-read. Uses `GITHUB_TOKEN` when set; GitHub-hosted upstreams
+only.
+
 ### validation.py
 
 Validation index and ground truth formatting. Used by verify.py for emulator-level checks