feat: add profile sync against upstream

scripts/upstream.py

@@ -0,0 +1,420 @@
+"""Access to upstream source repositories.
+
+Resolves revisions, fetches files by sha, and compares trees across the
+forge families the emulator profiles point at. Knows nothing about profile
+structure.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+USER_AGENT = "retrobios-profile-sync/1.0"
+ABSENT = "\0absent\0"
+GITHUB_COMPARE_CAP = 300
+
+_HOSTS: dict[str, tuple[str, str, str]] = {
+    "github.com": (
+        "github",
+        "https://api.github.com",
+        "https://raw.githubusercontent.com",
+    ),
+    "gitlab.com": ("gitlab", "https://gitlab.com/api/v4", "https://gitlab.com"),
+    "codeberg.org": (
+        "forgejo",
+        "https://codeberg.org/api/v1",
+        "https://codeberg.org",
+    ),
+    "git.citron-emu.org": (
+        "forgejo",
+        "https://git.citron-emu.org/api/v1",
+        "https://git.citron-emu.org",
+    ),
+    "git.eden-emu.dev": (
+        "forgejo",
+        "https://git.eden-emu.dev/api/v1",
+        "https://git.eden-emu.dev",
+    ),
+}
+
+
+class UpstreamError(Exception):
+    """Any failure while talking to a forge."""
+
+
+class RateLimitError(UpstreamError):
+    """The forge refused the request for quota reasons."""
+
+
+@dataclass(frozen=True)
+class Repo:
+    host: str
+    family: str
+    api_base: str
+    raw_base: str
+    owner: str
+    name: str
+
+    @property
+    def slug(self) -> str:
+        return f"{self.owner}/{self.name}"
+
+
+@dataclass(frozen=True)
+class Release:
+    tag: str
+    date: str
+    is_prerelease: bool
+
+
+@dataclass(frozen=True)
+class FileChange:
+    status: str
+    path: str
+    previous_path: str | None
+
+
+@dataclass(frozen=True)
+class CompareResult:
+    files: list[FileChange]
+    truncated: bool
+
+
+def parse_repo(url: str) -> Repo | None:
+    """Build a Repo from a forge URL, or None when the host is unknown."""
+    if not url:
+        return None
+    parts = urllib.parse.urlsplit(url.strip())
+    entry = _HOSTS.get(parts.netloc)
+    if entry is None:
+        return None
+    segments = [s for s in parts.path.split("/") if s]
+    if len(segments) < 2:
+        return None
+    owner, name = segments[0], segments[1]
+    if name.endswith(".git"):
+        name = name[:-4]
+    family, api_base, raw_base = entry
+    return Repo(parts.netloc, family, api_base, raw_base, owner, name)
+
+
+def make_repo(host: str, owner: str, name: str) -> Repo | None:
+    """Rebuild a Repo from a host and slug already known to be supported."""
+    entry = _HOSTS.get(host)
+    if entry is None:
+        return None
+    family, api_base, raw_base = entry
+    return Repo(host, family, api_base, raw_base, owner, name)
+
+
+def raw_url(repo: Repo, sha: str, path: str) -> str:
+    """URL serving the raw bytes of one path at one revision."""
+    quoted = urllib.parse.quote(path)
+    if repo.family == "github":
+        return f"{repo.raw_base}/{repo.owner}/{repo.name}/{sha}/{quoted}"
+    if repo.family == "gitlab":
+        return f"{repo.raw_base}/{repo.owner}/{repo.name}/-/raw/{sha}/{quoted}"
+    return f"{repo.raw_base}/{repo.owner}/{repo.name}/raw/commit/{sha}/{quoted}"
+
+
+def _headers(accept_json: bool = False) -> dict[str, str]:
+    headers = {"User-Agent": USER_AGENT}
+    if accept_json:
+        headers["Accept"] = "application/json"
+    token = os.environ.get("GITHUB_TOKEN", "")
+    if token:
+        headers["Authorization"] = f"token {token}"
+    return headers
+
+
+def _http_text(url: str) -> str | None:
+    """Body of a GET, or None on 404. Replaced in tests."""
+    req = urllib.request.Request(url, headers=_headers())
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return resp.read().decode("utf-8", errors="replace")
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            return None
+        if exc.code in (403, 429):
+            raise RateLimitError(f"{url}: HTTP {exc.code}") from exc
+        raise UpstreamError(f"{url}: HTTP {exc.code}") from exc
+    except urllib.error.URLError as exc:
+        raise UpstreamError(f"{url}: {exc.reason}") from exc
+
+
+def _http_json(url: str) -> object | None:
+    """Parsed JSON body of a GET, or None on 404. Replaced in tests."""
+    req = urllib.request.Request(url, headers=_headers(accept_json=True))
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            return None
+        if exc.code in (403, 429):
+            raise RateLimitError(f"{url}: HTTP {exc.code}") from exc
+        raise UpstreamError(f"{url}: HTTP {exc.code}") from exc
+    except urllib.error.URLError as exc:
+        raise UpstreamError(f"{url}: {exc.reason}") from exc
+
+
+def cache_path(cache_dir: str, repo: Repo, sha: str, path: str) -> Path:
+    """Content-addressed location for one path at one revision."""
+    root = Path(cache_dir) / repo.host / repo.owner / repo.name / sha
+    safe = path.replace("\\", "/").strip("/")
+    target = (root / safe).resolve()
+    base = root.resolve()
+    if not str(target).startswith(str(base) + os.sep) and target != base:
+        target = base / safe.replace("/", "_").replace("..", "_")
+    return target
+
+
+def write_cache(target: Path, text: str) -> None:
+    """Atomic write: unique scratch in the target directory, then replace."""
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, scratch = tempfile.mkstemp(dir=str(target.parent), suffix=".part")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(text)
+        os.replace(scratch, target)
+    except OSError:
+        Path(scratch).unlink(missing_ok=True)
+        raise
+
+
+def fetch_file(
+    repo: Repo,
+    sha: str,
+    path: str,
+    cache_dir: str,
+    offline: bool = False,
+) -> list[str] | None:
+    """Lines of one path at one revision, or None when absent."""
+    target = cache_path(cache_dir, repo, sha, path)
+    if target.is_file():
+        text = target.read_text(encoding="utf-8")
+        return None if text == ABSENT else text.splitlines()
+    if offline:
+        return None
+    text = _http_text(raw_url(repo, sha, path))
+    write_cache(target, ABSENT if text is None else text)
+    return None if text is None else text.splitlines()
+
+
+def _api(url: str, cache_dir: str, offline: bool) -> object | None:
+    """Cached API call. The cache stores absence as well as payloads."""
+    key = hashlib.sha256(url.encode()).hexdigest()
+    target = Path(cache_dir) / "_api" / f"{key}.json"
+    if target.is_file():
+        raw = target.read_text(encoding="utf-8")
+        return None if raw == ABSENT else json.loads(raw)
+    if offline:
+        return None
+    payload = _http_json(url)
+    write_cache(target, ABSENT if payload is None else json.dumps(payload))
+    return payload
+
+
+def _project(repo: Repo) -> str:
+    return urllib.parse.quote(f"{repo.owner}/{repo.name}", safe="")
+
+
+def _commits_url(repo: Repo, date: str | None) -> str:
+    if repo.family == "github":
+        base = f"{repo.api_base}/repos/{repo.slug}/commits?per_page=1"
+    elif repo.family == "gitlab":
+        base = (
+            f"{repo.api_base}/projects/{_project(repo)}"
+            f"/repository/commits?per_page=1"
+        )
+    else:
+        base = f"{repo.api_base}/repos/{repo.slug}/commits?limit=1"
+    return f"{base}&until={date}T23:59:59Z" if date else base
+
+
+def _first_sha(payload: object) -> str | None:
+    if isinstance(payload, list) and payload:
+        head = payload[0]
+        if isinstance(head, dict):
+            return head.get("sha") or head.get("id")
+    return None
+
+
+def resolve_head(repo: Repo, cache_dir: str, offline: bool = False) -> str | None:
+    """Sha of the default branch tip."""
+    return _first_sha(_api(_commits_url(repo, None), cache_dir, offline))
+
+
+def resolve_commit_at(
+    repo: Repo, date: str, cache_dir: str, offline: bool = False
+) -> str | None:
+    """Last default-branch commit on or before a date."""
+    return _first_sha(_api(_commits_url(repo, date), cache_dir, offline))
+
+
+def _tags_url(repo: Repo) -> str:
+    if repo.family == "gitlab":
+        return f"{repo.api_base}/projects/{_project(repo)}/repository/tags"
+    return f"{repo.api_base}/repos/{repo.slug}/tags"
+
+
+def list_tags(repo: Repo, cache_dir: str, offline: bool = False) -> list[str]:
+    """Tag names, newest first as the forge orders them."""
+    payload = _api(_tags_url(repo), cache_dir, offline)
+    if not isinstance(payload, list):
+        return []
+    return [t["name"] for t in payload if isinstance(t, dict) and t.get("name")]
+
+
+def resolve_tag_commit(
+    repo: Repo, tag: str, cache_dir: str, offline: bool = False
+) -> str | None:
+    """Commit a tag points at."""
+    payload = _api(_tags_url(repo), cache_dir, offline)
+    if not isinstance(payload, list):
+        return None
+    for entry in payload:
+        if not isinstance(entry, dict) or entry.get("name") != tag:
+            continue
+        commit = entry.get("commit")
+        if isinstance(commit, dict):
+            return commit.get("sha") or commit.get("id")
+    return None
+
+
+def _releases_url(repo: Repo) -> str:
+    if repo.family == "gitlab":
+        return f"{repo.api_base}/projects/{_project(repo)}/releases"
+    return f"{repo.api_base}/repos/{repo.slug}/releases/latest"
+
+
+def latest_release(
+    repo: Repo, cache_dir: str, offline: bool = False
+) -> Release | None:
+    """Most recent release the forge exposes."""
+    payload = _api(_releases_url(repo), cache_dir, offline)
+    if isinstance(payload, list):
+        payload = payload[0] if payload else None
+    if not isinstance(payload, dict):
+        return None
+    tag = payload.get("tag_name") or payload.get("tag") or ""
+    stamp = payload.get("published_at") or payload.get("released_at") or ""
+    if not tag:
+        return None
+    return Release(str(tag), str(stamp)[:10], bool(payload.get("prerelease")))
+
+
+def _compare_url(repo: Repo, base: str, head: str) -> str:
+    if repo.family == "gitlab":
+        return (
+            f"{repo.api_base}/projects/{_project(repo)}"
+            f"/repository/compare?from={base}&to={head}"
+        )
+    return f"{repo.api_base}/repos/{repo.slug}/compare/{base}...{head}"
+
+
+def _changes_from_github(payload: dict) -> list[FileChange]:
+    return [
+        FileChange(
+            entry.get("status", "modified"),
+            entry.get("filename", ""),
+            entry.get("previous_filename"),
+        )
+        for entry in (payload.get("files") or [])
+        if isinstance(entry, dict)
+    ]
+
+
+def _changes_from_gitlab(payload: dict) -> list[FileChange]:
+    changes = []
+    for entry in payload.get("diffs") or []:
+        if not isinstance(entry, dict):
+            continue
+        if entry.get("renamed_file"):
+            status = "renamed"
+        elif entry.get("new_file"):
+            status = "added"
+        elif entry.get("deleted_file"):
+            status = "removed"
+        else:
+            status = "modified"
+        previous = entry.get("old_path") if status == "renamed" else None
+        changes.append(FileChange(status, entry.get("new_path", ""), previous))
+    return changes
+
+
+def compare(
+    repo: Repo, base: str, head: str, cache_dir: str, offline: bool = False
+) -> CompareResult:
+    """Tree difference between two revisions."""
+    payload = _api(_compare_url(repo, base, head), cache_dir, offline)
+    if not isinstance(payload, dict):
+        return CompareResult([], True)
+    if repo.family == "gitlab":
+        files = _changes_from_gitlab(payload)
+    else:
+        files = _changes_from_github(payload)
+    truncated = bool(payload.get("truncated")) or len(files) >= GITHUB_COMPARE_CAP
+    return CompareResult(files, truncated)
+
+
+def find_renamed(result: CompareResult, path: str) -> str | None:
+    """New path of a file the comparison reports as renamed."""
+    for change in result.files:
+        if change.status == "renamed" and change.previous_path == path:
+            return change.path
+    return None
+
+
+def _tree_url(repo: Repo, sha: str) -> str | None:
+    if repo.family == "gitlab":
+        return None
+    return f"{repo.api_base}/repos/{repo.slug}/git/trees/{sha}?recursive=1"
+
+
+def list_tree(
+    repo: Repo, sha: str, cache_dir: str, offline: bool = False
+) -> tuple[list[str], bool]:
+    """Every blob path at one revision, and whether the forge truncated it."""
+    url = _tree_url(repo, sha)
+    if url is None:
+        return [], True
+    payload = _api(url, cache_dir, offline)
+    if not isinstance(payload, dict):
+        return [], True
+    paths = [
+        entry["path"]
+        for entry in payload.get("tree") or []
+        if isinstance(entry, dict) and entry.get("type") == "blob" and entry.get("path")
+    ]
+    return paths, bool(payload.get("truncated"))
+
+
+def commits_touching(
+    repo: Repo, path: str, base: str, cache_dir: str, offline: bool = False
+) -> int:
+    """Commits touching one path since a revision."""
+    quoted = urllib.parse.quote(path)
+    if repo.family == "gitlab":
+        url = (
+            f"{repo.api_base}/projects/{_project(repo)}"
+            f"/repository/commits?path={quoted}&per_page=100"
+        )
+    elif repo.family == "github":
+        url = (
+            f"{repo.api_base}/repos/{repo.slug}/commits"
+            f"?path={quoted}&sha={base}&per_page=100"
+        )
+    else:
+        url = f"{repo.api_base}/repos/{repo.slug}/commits?path={quoted}&limit=100"
+    payload = _api(url, cache_dir, offline)
+    return len(payload) if isinstance(payload, list) else 0

tests/test_upstream.py

@@ -0,0 +1,318 @@
+"""Tests for the upstream repository access module (no network)."""
+
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
+
+import upstream
+from upstream import make_repo, parse_repo, raw_url
+
+
+def _no_network(url: str):
+    """Any call reaching here is a leak: the suite must stay offline."""
+    raise AssertionError(f"test reached the network: {url}")
+
+
+class TestParseRepo(unittest.TestCase):
+    def test_github(self):
+        repo = parse_repo("https://github.com/libretro/beetle-psx-libretro")
+        self.assertEqual(repo.family, "github")
+        self.assertEqual(repo.owner, "libretro")
+        self.assertEqual(repo.name, "beetle-psx-libretro")
+
+    def test_github_trailing_git_and_slash(self):
+        self.assertEqual(
+            parse_repo("https://github.com/mamedev/mame.git/"),
+            parse_repo("https://github.com/mamedev/mame"),
+        )
+
+    def test_gitlab(self):
+        repo = parse_repo("https://gitlab.com/recalbox/recalbox")
+        self.assertEqual(repo.family, "gitlab")
+
+    def test_codeberg_is_forgejo(self):
+        self.assertEqual(parse_repo("https://codeberg.org/a/b").family, "forgejo")
+
+    def test_known_forgejo_instances(self):
+        for url in (
+            "https://git.citron-emu.org/citron/emu",
+            "https://git.eden-emu.dev/eden-emu/eden",
+        ):
+            self.assertEqual(parse_repo(url).family, "forgejo")
+
+    def test_unsupported_host(self):
+        self.assertIsNone(parse_repo("https://sourceforge.net/projects/vice"))
+
+    def test_non_repository_url(self):
+        self.assertIsNone(parse_repo("https://mednafen.github.io/"))
+
+    def test_empty(self):
+        self.assertIsNone(parse_repo(""))
+
+
+class TestRawUrl(unittest.TestCase):
+    def test_github_raw(self):
+        repo = parse_repo("https://github.com/libretro/x")
+        self.assertEqual(
+            raw_url(repo, "abc123", "src/main.cpp"),
+            "https://raw.githubusercontent.com/libretro/x/abc123/src/main.cpp",
+        )
+
+    def test_gitlab_raw(self):
+        repo = parse_repo("https://gitlab.com/g/p")
+        self.assertEqual(
+            raw_url(repo, "abc123", "a.c"),
+            "https://gitlab.com/g/p/-/raw/abc123/a.c",
+        )
+
+    def test_forgejo_raw(self):
+        repo = parse_repo("https://codeberg.org/g/p")
+        self.assertEqual(
+            raw_url(repo, "abc123", "a.c"),
+            "https://codeberg.org/g/p/raw/commit/abc123/a.c",
+        )
+
+    def test_path_is_quoted(self):
+        repo = parse_repo("https://github.com/o/n")
+        self.assertIn("src/a%20b.cpp", raw_url(repo, "s", "src/a b.cpp"))
+
+
+class TestMakeRepo(unittest.TestCase):
+    def test_rebuilds_a_known_host(self):
+        self.assertEqual(
+            make_repo("github.com", "o", "n"), parse_repo("https://github.com/o/n")
+        )
+
+    def test_unknown_host_returns_none(self):
+        self.assertIsNone(make_repo("example.invalid", "o", "n"))
+
+
+class TestCache(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = self.tmp.name
+        self.repo = parse_repo("https://github.com/o/n")
+        self.calls: list[str] = []
+        self._orig = (upstream._http_text, upstream._http_json)
+        upstream._http_text = self._fake
+        upstream._http_json = _no_network
+
+    def tearDown(self):
+        upstream._http_text, upstream._http_json = self._orig
+        self.tmp.cleanup()
+
+    def _fake(self, url: str) -> str | None:
+        self.calls.append(url)
+        return "line one\nline two\n"
+
+    def test_path_includes_host_slug_and_sha(self):
+        p = upstream.cache_path(self.dir, self.repo, "abc123", "src/a.cpp")
+        self.assertIn("github.com", str(p))
+        self.assertIn("o", str(p))
+        self.assertIn("abc123", str(p))
+
+    def test_path_is_contained_in_cache_dir(self):
+        p = upstream.cache_path(self.dir, self.repo, "abc", "../../escape.c")
+        self.assertTrue(str(p.resolve()).startswith(str(Path(self.dir).resolve())))
+
+    def test_fetch_then_cache_hit(self):
+        first = upstream.fetch_file(self.repo, "abc", "a.c", self.dir)
+        second = upstream.fetch_file(self.repo, "abc", "a.c", self.dir)
+        self.assertEqual(first, ["line one", "line two"])
+        self.assertEqual(second, first)
+        self.assertEqual(len(self.calls), 1)
+
+    def test_offline_miss_returns_none_without_request(self):
+        self.assertIsNone(
+            upstream.fetch_file(self.repo, "abc", "a.c", self.dir, offline=True)
+        )
+        self.assertEqual(self.calls, [])
+
+    def test_offline_hit_serves_cache(self):
+        upstream.fetch_file(self.repo, "abc", "a.c", self.dir)
+        self.calls.clear()
+        self.assertEqual(
+            upstream.fetch_file(self.repo, "abc", "a.c", self.dir, offline=True),
+            ["line one", "line two"],
+        )
+        self.assertEqual(self.calls, [])
+
+    def test_missing_file_is_cached_as_absent(self):
+        upstream._http_text = lambda url: None
+        self.assertIsNone(upstream.fetch_file(self.repo, "abc", "gone.c", self.dir))
+        self.assertIsNone(
+            upstream.fetch_file(self.repo, "abc", "gone.c", self.dir, offline=True)
+        )
+
+    def test_write_leaves_no_temporary_behind(self):
+        target = Path(self.dir) / "sub" / "f.txt"
+        upstream.write_cache(target, "payload")
+        self.assertEqual(target.read_text(encoding="utf-8"), "payload")
+        siblings = list(target.parent.iterdir())
+        self.assertEqual([p.name for p in siblings], ["f.txt"])
+
+
+class TestRevisions(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = self.tmp.name
+        self.repo = parse_repo("https://github.com/o/n")
+        self.responses: dict[str, object] = {}
+        self.calls: list[str] = []
+        self._orig = (upstream._http_json, upstream._http_text)
+        upstream._http_json = self._fake
+        upstream._http_text = _no_network
+
+    def tearDown(self):
+        upstream._http_json, upstream._http_text = self._orig
+        self.tmp.cleanup()
+
+    def _fake(self, url: str):
+        self.calls.append(url)
+        for fragment, payload in self.responses.items():
+            if fragment in url:
+                return payload
+        return None
+
+    def test_resolve_head_github(self):
+        self.responses["/commits"] = [{"sha": "deadbeef"}]
+        self.assertEqual(upstream.resolve_head(self.repo, self.dir), "deadbeef")
+
+    def test_resolve_head_gitlab_uses_id(self):
+        repo = parse_repo("https://gitlab.com/g/p")
+        self.responses["/repository/commits"] = [{"id": "cafe"}]
+        self.assertEqual(upstream.resolve_head(repo, self.dir), "cafe")
+
+    def test_resolve_commit_at_passes_date(self):
+        self.responses["/commits"] = [{"sha": "abc"}]
+        self.assertEqual(
+            upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir), "abc"
+        )
+        self.assertIn("2026-03-29", self.calls[0])
+
+    def test_api_response_is_cached(self):
+        self.responses["/commits"] = [{"sha": "abc"}]
+        upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir)
+        upstream.resolve_commit_at(self.repo, "2026-03-29", self.dir)
+        self.assertEqual(len(self.calls), 1)
+
+    def test_offline_without_cache_returns_none(self):
+        self.assertIsNone(upstream.resolve_head(self.repo, self.dir, offline=True))
+        self.assertEqual(self.calls, [])
+
+    def test_empty_history_returns_none(self):
+        self.responses["/commits"] = []
+        self.assertIsNone(upstream.resolve_head(self.repo, self.dir))
+
+    def test_list_tags(self):
+        self.responses["/tags"] = [{"name": "v1.2"}, {"name": "v1.1"}]
+        self.assertEqual(upstream.list_tags(self.repo, self.dir), ["v1.2", "v1.1"])
+
+    def test_resolve_tag_commit(self):
+        self.responses["/tags"] = [{"name": "v1.2", "commit": {"sha": "tagsha"}}]
+        self.assertEqual(
+            upstream.resolve_tag_commit(self.repo, "v1.2", self.dir), "tagsha"
+        )
+
+    def test_latest_release(self):
+        self.responses["/releases/latest"] = {
+            "tag_name": "v3.0.0",
+            "published_at": "2026-06-14T10:00:00Z",
+            "prerelease": False,
+        }
+        rel = upstream.latest_release(self.repo, self.dir)
+        self.assertEqual(rel.tag, "v3.0.0")
+        self.assertEqual(rel.date, "2026-06-14")
+        self.assertFalse(rel.is_prerelease)
+
+    def test_missing_release_returns_none(self):
+        self.assertIsNone(upstream.latest_release(self.repo, self.dir))
+
+    def test_rate_limit_propagates(self):
+        def boom(url: str):
+            raise upstream.RateLimitError("quota")
+
+        upstream._http_json = boom
+        with self.assertRaises(upstream.RateLimitError):
+            upstream.resolve_head(self.repo, self.dir)
+
+
+class TestCompare(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = self.tmp.name
+        self.repo = parse_repo("https://github.com/o/n")
+        self.payload: object = None
+        self._orig = (upstream._http_json, upstream._http_text)
+        upstream._http_json = lambda url: self.payload
+        upstream._http_text = _no_network
+
+    def tearDown(self):
+        upstream._http_json, upstream._http_text = self._orig
+        self.tmp.cleanup()
+
+    def test_renamed_file_carries_previous_path(self):
+        self.payload = {
+            "files": [
+                {
+                    "status": "renamed",
+                    "filename": "src/new.cpp",
+                    "previous_filename": "old.cpp",
+                },
+                {"status": "modified", "filename": "a.c"},
+            ]
+        }
+        result = upstream.compare(self.repo, "a", "b", self.dir)
+        self.assertFalse(result.truncated)
+        self.assertEqual(upstream.find_renamed(result, "old.cpp"), "src/new.cpp")
+
+    def test_no_rename_returns_none(self):
+        self.payload = {"files": [{"status": "modified", "filename": "a.c"}]}
+        result = upstream.compare(self.repo, "a", "b", self.dir)
+        self.assertIsNone(upstream.find_renamed(result, "old.cpp"))
+
+    def test_truncated_at_github_cap(self):
+        self.payload = {
+            "files": [
+                {"status": "modified", "filename": f"f{i}.c"} for i in range(300)
+            ]
+        }
+        self.assertTrue(upstream.compare(self.repo, "a", "b", self.dir).truncated)
+
+    def test_missing_comparison_is_empty_and_truncated(self):
+        self.payload = None
+        result = upstream.compare(self.repo, "a", "b", self.dir)
+        self.assertEqual(result.files, [])
+        self.assertTrue(result.truncated)
+
+    def test_gitlab_diffs_shape(self):
+        repo = parse_repo("https://gitlab.com/g/p")
+        self.payload = {
+            "diffs": [
+                {
+                    "new_path": "new.c",
+                    "old_path": "old.c",
+                    "renamed_file": True,
+                    "new_file": False,
+                    "deleted_file": False,
+                }
+            ]
+        }
+        result = upstream.compare(repo, "a", "b", self.dir)
+        self.assertEqual(upstream.find_renamed(result, "old.c"), "new.c")
+
+    def test_commits_touching_counts_entries(self):
+        self.payload = [{"sha": "1"}, {"sha": "2"}, {"sha": "3"}]
+        self.assertEqual(
+            upstream.commits_touching(self.repo, "a.c", "base", self.dir), 3
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/architecture.md

@@ -275,19 +275,23 @@ user's platform, filter files by hardware target, and download with SHA1 verific
 
 ## Tests
 
-10 test files, 400 tests total:
+14 test files, 586 tests total:
 
 | File | Tests | Coverage |
 |------|-------|----------|
-| `test_e2e.py` | 217 | file resolution, verification, severity, cross-reference, aliases, inheritance, shared groups, data dirs, storage tiers, HLE, launchers, platform grouping, core resolution, target filtering, truth/diff, exporters |
+| `test_e2e.py` | 218 | file resolution, verification, severity, cross-reference, aliases, inheritance, shared groups, data dirs, storage tiers, HLE, launchers, platform grouping, core resolution, target filtering, truth/diff, exporters |
+| `test_profile_sync.py` | 129 | ref anchoring, guarded profile writes, detection, triage |
 | `test_install.py` | 70 | `install.py` platform detection, config-file parsing, manifest handling |
+| `test_upstream.py` | 38 | forge URL parsing, cache, revision resolution, tree comparison |
 | `test_provenance.py` | 29 | Logiqx/Redump parsers, DAT pack import, provenance join, coverage report |
-| `test_mame_parser.py` | 22 | BIOS root set detection, ROM block parsing, macro expansion |
+| `test_mame_parser.py` | 25 | BIOS root set detection, ROM block parsing, macro expansion |
 | `test_hash_merge.py` | 17 | MAME/FBNeo YAML merge, diff detection, formatting preservation |
 | `test_fbneo_parser.py` | 16 | BIOS set detection, ROM info parsing |
-| `test_profile_refs.py` | 12 | `check_profile_refs` pure functions (no network) |
+| `test_deterministic_zip.py` | 12 | streaming rebuild, metadata normalisation, entry ordering, source CRC |
+| `test_artifact_lock.py` | 10 | writer/writer and writer/reader exclusion, reader sharing, release on error |
 | `test_pack_integrity.py` | 8 | extract ZIP packs to disk, verify paths + hashes per platform's native mode |
 | `test_torrentzip.py` | 8 | TorrentZip builder against real MAME romsets |
+| `test_large_file_cache.py` | 5 | concurrent downloads, temporary file residue, hash rejection |
 | `test_no_case_collisions.py` | 1 | guard against case-colliding paths in `bios/` |
 
 ```bash
@@ -295,9 +299,9 @@ python -m unittest discover tests -v      # full suite
 python -m unittest tests.test_e2e -v      # single module
 ```
 
-`test_e2e.py`, `test_install.py`, `test_provenance.py`, the parser tests and
-`test_profile_refs.py` run on synthetic fixtures with no network and no real
-BIOS files. `test_pack_integrity.py`, `test_torrentzip.py` and
+`test_e2e.py`, `test_install.py`, `test_provenance.py`, the parser tests,
+`test_profile_sync.py` and `test_upstream.py` run on synthetic fixtures with no
+network and no real BIOS files. `test_pack_integrity.py`, `test_torrentzip.py` and
 `test_no_case_collisions.py` read the working tree and skip when the data they
 need is absent. See the [testing guide](testing-guide.md) for the fixture
 pattern and how to add a test.

wiki/profiling.md

@@ -215,7 +215,7 @@ A few field conventions that protect the toolchain:
 python scripts/cross_reference.py --emulator dolphin --json
 python scripts/verify.py --emulator dolphin
 python scripts/verify.py --emulator dolphin --verbose   # per-core checks + source refs
-python scripts/check_profile_refs.py --emulator dolphin # do the source_ref lines still hold
+python scripts/profile_sync.py --emulator dolphin       # do the source_ref lines still hold
 ```
 
 The profile also has to satisfy `schemas/emulator.schema.json`, which CI checks

wiki/testing-guide.md

@@ -23,7 +23,11 @@ python -m unittest tests.test_provenance -v
 python -m unittest tests.test_mame_parser -v
 python -m unittest tests.test_hash_merge -v
 python -m unittest tests.test_fbneo_parser -v
-python -m unittest tests.test_profile_refs -v
+python -m unittest tests.test_profile_sync -v
+python -m unittest tests.test_upstream -v
+python -m unittest tests.test_deterministic_zip -v
+python -m unittest tests.test_artifact_lock -v
+python -m unittest tests.test_large_file_cache -v
 python -m unittest tests.test_pack_integrity -v
 python -m unittest tests.test_torrentzip -v
 python -m unittest tests.test_no_case_collisions -v
@@ -36,15 +40,19 @@ library `unittest` module.
 
 | Module | Tests | Fixtures | What it covers |
 |--------|-------|----------|----------------|
-| `test_e2e.py` | 217 | synthetic | resolution, verification, packs, cross-reference, targets, truth |
+| `test_e2e.py` | 218 | synthetic | resolution, verification, packs, cross-reference, targets, truth |
+| `test_profile_sync.py` | 129 | synthetic | ref anchoring, guarded profile writes, detection, triage |
 | `test_install.py` | 70 | synthetic | `install.py` detection, config parsing, manifest handling |
+| `test_upstream.py` | 38 | synthetic | forge URL parsing, cache, revision resolution, tree comparison |
 | `test_provenance.py` | 29 | synthetic | Logiqx/Redump parsing, DAT import, provenance join, coverage report |
-| `test_mame_parser.py` | 22 | inline C | BIOS root sets, ROM blocks, macro expansion |
+| `test_mame_parser.py` | 25 | inline C | BIOS root sets, ROM blocks, macro expansion |
 | `test_hash_merge.py` | 17 | synthetic | YAML hash merge, diff, formatting preservation |
 | `test_fbneo_parser.py` | 16 | inline C | `BDF_BOARDROM` sets, ROM info parsing |
-| `test_profile_refs.py` | 12 | synthetic | `check_profile_refs` pure functions, no network |
+| `test_deterministic_zip.py` | 12 | synthetic | streaming rebuild, metadata normalisation, entry ordering, source CRC |
+| `test_artifact_lock.py` | 10 | synthetic | writer/writer and writer/reader exclusion, reader sharing, release on error |
 | `test_pack_integrity.py` | 8 | real packs | extract each ZIP, verify paths and hashes |
 | `test_torrentzip.py` | 8 | real romsets | TorrentZip builder byte-for-byte |
+| `test_large_file_cache.py` | 5 | synthetic | concurrent downloads, temporary file residue, hash rejection |
 | `test_no_case_collisions.py` | 1 | real `bios/` | no case-colliding paths on Windows/macOS clones |
 
 ## Test architecture
@@ -123,9 +131,15 @@ detection, each registry detection method (`config_file`, `path_exists`,
 `file_exists`), config-file key parsing, manifest loading, target filtering,
 and destination resolution.
 
-**test_profile_refs.** Covers the pure functions of `check_profile_refs`
-(anchor matching, line-window search, hash extraction). The GitHub fetching
-path is not exercised, so the module runs offline.
+**test_profile_sync.** Covers ref anchoring end to end: the six statuses,
+widening a one-line anchor until it is unique, refusing to rebase an ambiguous
+one, following a rename, and the guarded YAML writes. Network access is
+replaced by an injected fetch function.
+
+**test_upstream.** Covers forge URL parsing for GitHub, GitLab and Forgejo,
+the content-addressed cache and its atomic write, revision and tag resolution,
+and tree comparison. The HTTP layer is replaced at module level, so nothing
+leaves the machine.
 
 ### Tests that read the working tree
 

wiki/tools.md

@@ -208,25 +208,55 @@ to the files inside the archive, so the exported DAT lists those entries
 without a container sha1. Anyone submitting the DAT upstream should mention
 this.
 
-### check_profile_refs.py
+### profile_sync.py
 
-Audit `source_ref` line references against the profiled upstream. The
-commit under audit is the profile's `source_commit` when present, else the
-last upstream commit at `profiled_date`. Each referenced file is fetched
-at that commit and at HEAD, and the entry's declared hashes (or filename)
-are searched around the cited lines.
+Confront a profile with its upstream. The pinned commit is the profile's
+`source_commit` when present, else the last upstream commit at
+`profiled_date`. Each cited line range is extracted at the pin and located
+in the HEAD revision of the same file.
 
 ```bash
-python scripts/check_profile_refs.py --emulator vice
-python scripts/check_profile_refs.py --all --json
+python scripts/profile_sync.py --emulator vice
+python scripts/profile_sync.py --emulator vice --full-diff
+python scripts/profile_sync.py --all --triage
+python scripts/profile_sync.py --all --changed-only --json
+python scripts/profile_sync.py --emulator vice --fetch-plan
 ```
 
-Per ref and revision: `anchored` (found at the cited lines), `moved`
-(found elsewhere in the file), `gone` (absent from the file). `moved` at
-HEAD means upstream shifted since profiling; `gone` at pin means the
-declared value does not come from the referenced file and the entry
-deserves a re-read. Uses `GITHUB_TOKEN` when set; GitHub-hosted upstreams
-only.
+Per part of a ref: `ANCHORED` (same content, same lines), `SHIFTED` (same
+content, moved), `RENAMED` (the source file moved), `CHANGED` (content
+edited), `AMBIGUOUS` (several equally good candidates), `GONE` (nothing
+left to anchor to). An entry carries the worst status of its parts.
+
+A single cited line is often not distinctive, so the anchor widens by
+steps of ±3, ±6 and ±12 lines until it is unique. Anything still ambiguous
+is reported, never guessed.
+
+Three shorthand forms appear in the corpus and are resolved rather than
+reported missing. A part reduced to a line range continues the previous
+part's file (`geo.c:234-243, 273-285`). A profile whose `source` differs
+from its `upstream` may cite paths from both, and each path is attributed
+to the repository that carries it. A path prefixed with a repository
+directory name (`EightyOne/Source/HW_.cpp`) is stripped as a last resort,
+only after the path as written has failed everywhere, and the result is
+reported as `RENAMED` so `--rebase-refs` cleans the profile.
+
+`--check-version` compares `core_version` with the latest upstream tag and
+release. `--detect-new-files` lists filename literals at HEAD the profile
+does not declare. `--watch-hashes` lists hash literals added upstream that
+match no entry. `--tree-diff` shows added, removed and renamed files in the
+directories the refs point at.
+
+Writes are explicit and mechanical only. `--backfill-commits` fills a
+missing `source_commit`, `--rebase-refs` recales `SHIFTED` and `RENAMED`
+line ranges, `--bump-commit` advances `source_commit` to HEAD only when
+nothing needs a re-read. All three refuse to run on a dirty `emulators/`
+without `--force`, and every write is verified by reparsing the document.
+
+Uses `GITHUB_TOKEN` when set, which `--all` requires. Responses are cached
+under `.cache/upstream/`, addressed by commit sha, so `--offline` replays a
+previous run. GitHub, GitLab and Forgejo upstreams; other hosts are
+reported as skipped.
 
 ### validation.py