feat: link only source paths a tree confirms

The deploy has no tree cache, so a citation it could not check was
linked into the declared repository: 30 paths answered 404. The paths
held at each pin are now recorded in website/src/data/source-paths.json
by python scripts/sitesources.py, and a path absent from it renders as
code. An address inside a shipped binary reads as one.

scripts/sitesources.py

@@ -5,17 +5,21 @@ was written against, so a permalink is pinned, never guessed."""
 
 from __future__ import annotations
 
+import argparse
 import functools
+import json
 import re
+import sys
 import urllib.parse
+from collections.abc import Iterator
 from dataclasses import dataclass
 from pathlib import Path
-import urllib.request
 
 import upstream
 from siterender import _by_mode
 from sourceref import (
-    _prose_runs, is_binary_citation, ref_role, source_ref_values, split_source_ref,
+    _HEX_ADDRESS_RE, _prose_runs, is_binary_citation, ref_role, source_ref_values,
+    split_source_ref,
 )
 
 
@@ -154,43 +158,159 @@ def _source_permalink(repo: upstream.Repo, pin: str, path: str,
                 url += f"-L{end}"
     return url
 
-# The tree listings profile_sync caches. Read offline only: a page build
-# never calls a forge, and a listing it does not find leaves the link as the
-# profile's declarations alone decide it.
+# The tree listings profile_sync caches, read only by the command that
+# writes SOURCE_PATHS. The pages read that file: the deploy has no tree
+# cache, and a citation it could not check was linked into whichever
+# repository the profile declared, 30 of them to a 404.
 TREE_CACHE = Path(__file__).resolve().parent.parent / upstream.CACHE_DIR
+SOURCE_PATHS = (Path(__file__).resolve().parent.parent
+                / "website" / "src" / "data" / "source-paths.json")
+
+
+def tree_key(repo: upstream.Repo, pin: str) -> str:
+    """How SOURCE_PATHS names one repository at one revision."""
+    return f"{repo.host}/{repo.slug}@{pin}"
 
 
 @functools.lru_cache(maxsize=None)
-def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str] | None:
-    """Every file of a repository at a revision, None when not cached or
-    truncated (a missing path would then prove nothing)."""
-    if not repo.api_base:
-        return None  # a browse-only forge: no listing to read
-    paths, truncated = upstream.list_tree(repo, pin, str(TREE_CACHE), offline=True)
-    return frozenset(paths) if paths and not truncated else None
+def _verified() -> dict[str, frozenset[str]]:
+    """The committed record, empty when it has not been written."""
+    try:
+        data = json.loads(SOURCE_PATHS.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {}
+    return {key: frozenset(paths) for key, paths in data.items()}
 
 
-def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool | None:
-    """Whether the repository holds ``path`` at ``pin``: None when its tree
-    is not cached. trident cites src/core/loader/ncch.cpp, a Panda3DS file,
-    and the page linked it into 3dsTrident, where it is a 404."""
-    paths = _tree_paths(repo, pin)
-    return None if paths is None else path in paths
+@functools.lru_cache(maxsize=None)
+def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str]:
+    """The cited paths of a repository at a revision that its tree holds."""
+    return _verified().get(tree_key(repo, pin), frozenset())
+
+
+def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool:
+    """Whether ``path`` was found in the repository's tree at ``pin``.
+
+    trident cites src/core/loader/ncch.cpp, a Panda3DS file, and the page
+    linked it into 3dsTrident, where it is a 404. A path no tree confirmed,
+    on a forge whose tree cannot be read or a revision not cached when the
+    record was written, is not linked: no link beats a link to nothing.
+    """
+    return path in _tree_paths(repo, pin)
 
 
 def _holder(
-    candidates: list[tuple[upstream.Repo, str]], path: str, default: tuple | None,
+    candidates: list[tuple[upstream.Repo, str]], path: str,
 ) -> tuple[upstream.Repo, str] | None:
     """The pinned repository that holds ``path``, source before upstream as
-    profile_sync.select_views orders them. Without every tree cached, the
-    declarations decide (``default``). A path no cached tree holds links
-    nowhere: no link beats a link to a file that is not there."""
-    verdicts = [tree_holds(repo, pin, path) for repo, pin in candidates]
-    if not candidates or None in verdicts:
-        if default is not None and tree_holds(*default, path) is False:
+    profile_sync.select_views orders them."""
+    return next((cand for cand in candidates if tree_holds(*cand, path)), None)
+
+
+def _cached_tree(
+    repo: upstream.Repo, pin: str, cache_dirs: list[Path], fetch: bool = False,
+) -> frozenset[str] | None:
+    """Every file of a repository at a revision, None when not cached or
+    truncated (a missing path would then prove nothing). Each cache is read
+    in turn. With *fetch*, a tree none holds is fetched into the last one."""
+    if not repo.api_base:
+        return None  # a browse-only forge: no listing to read
+    for index, cache_dir in enumerate(cache_dirs):
+        online = fetch and index == len(cache_dirs) - 1
+        try:
+            paths, truncated = upstream.list_tree(repo, pin, str(cache_dir), offline=not online)
+        except upstream.UpstreamError as exc:
+            print(f"  {tree_key(repo, pin)}: {exc}", file=sys.stderr)
             return None
-        return default
-    return next((cand for cand, held in zip(candidates, verdicts) if held), None)
+        if paths and not truncated:
+            return frozenset(paths)
+    return None
+
+
+def _walk(value: object) -> Iterator[tuple[str, object]]:
+    """Every (key, value) of a profile, nested lists and mappings included."""
+    if isinstance(value, dict):
+        for key, item in value.items():
+            yield str(key), item
+            yield from _walk(item)
+    elif isinstance(value, list):
+        for item in value:
+            yield from _walk(item)
+
+
+def cited_paths(profile: dict) -> set[str]:
+    """Every path a page of the profile could link: the parts of each
+    source_ref and the file:line citations of its prose."""
+    paths: set[str] = set()
+    for key, value in _walk(profile):
+        if key == "source_ref" and isinstance(value, (str, dict)):
+            for _label, refs in source_ref_values(value):
+                paths.update(ref_role(part.path)[1] for part in split_source_ref(refs))
+        elif isinstance(value, str) and ":" in value:
+            for _n, _span, _spans, parts, _run, _wrapped in _prose_runs(value):
+                paths.update(part.path for part in parts)
+    return {path for path in paths
+            if _PATH_RE.fullmatch(path) and not is_binary_citation(path)}
+
+
+def profile_trees(profile: dict) -> list[tuple[upstream.Repo, str]]:
+    """Every repository and revision a page of the profile can link into."""
+    found: list[tuple[upstream.Repo, str]] = []
+    labels = {""} | {
+        str(mode) for field in ("source", "upstream", "source_commit", "upstream_commit")
+        if isinstance(profile.get(field), dict) for mode in profile[field]}
+    for label in sorted(labels):
+        for pair in _forge_sources(profile, label) + list(role_sources(profile, label).values()):
+            if pair not in found:
+                found.append(pair)
+    return found
+
+
+def _listed_directories(
+    repo: upstream.Repo, pin: str, paths: set[str], cache_dirs: list[Path], fetch: bool,
+) -> frozenset[str] | None:
+    """The cited paths found by listing their directories one by one, for a
+    tree the forge truncates (aps3e vendors RPCS3 whole). A directory that
+    cannot be listed confirms none of its paths. None off GitHub, the one
+    forge with that call."""
+    if repo.family != "github":
+        return None
+    found: set[str] = set()
+    for folder in sorted({path.rpartition("/")[0] for path in paths} - {""}):
+        names = None
+        for index, cache_dir in enumerate(cache_dirs):
+            online = fetch and index == len(cache_dirs) - 1
+            try:
+                names = upstream.list_directory(repo, pin, folder, str(cache_dir), offline=not online)
+            except upstream.UpstreamError:
+                names = None
+            if names is not None:
+                break
+        found.update(f"{folder}/{name}" for name in names or ())
+    return frozenset(found)
+
+
+def build_source_paths(
+    profiles: dict, cache_dirs: list[Path], fetch: bool = False,
+) -> tuple[dict[str, list[str]], list[str]]:
+    """The cited paths each cached tree holds, and the trees not cached."""
+    held: dict[str, set[str]] = {}
+    unread: set[str] = set()
+    for _name, profile in sorted(profiles.items()):
+        paths = cited_paths(profile)
+        if not paths:
+            continue
+        for repo, pin in profile_trees(profile):
+            tree = _cached_tree(repo, pin, cache_dirs, fetch)
+            if tree is None:
+                tree = _listed_directories(repo, pin, paths, cache_dirs, fetch)
+            if tree is None:
+                unread.add(tree_key(repo, pin))
+                continue
+            found = paths & tree
+            if found:
+                held.setdefault(tree_key(repo, pin), set()).update(found)
+    return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread)
 
 
 def _commit_url(repo: upstream.Repo, pin: str) -> str:
@@ -279,11 +399,11 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
     """Render source_ref values as pinned links when their forge is known.
 
     A profile can declare two repositories: the libretro port in ``source`` and
-    the original emulator in ``upstream``. Which of the two carries a given
-    path cannot be decided without reading their trees, and this generator runs
-    offline. Rather than guess, an unattributable path is rendered as plain
+    the original emulator in ``upstream``. A path links into the one whose
+    tree SOURCE_PATHS says holds it. Any other path is rendered as plain
     code: a citation with no link still names the file and the lines, while a
-    link to the wrong repository is a false citation.
+    link to the wrong repository is a false citation. An address inside a
+    shipped binary reads as one (offset 0x8094-0x80ac in libnes.so).
     """
     rendered_groups: list[str] = []
     path_re = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
@@ -292,7 +412,13 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
         by_role = role_sources(profile, label)
         rendered: list[str] = []
         current_role = ""
+        binary = ""
         for part in split_source_ref(refs):
+            if is_binary_citation(part.path):
+                text, binary = _binary_citation(part.path, binary)
+                rendered.append(text)
+                continue
+            binary = ""
             role, bare = ref_role(part.path)
             if bare != part.path or role:
                 current_role = role
@@ -311,20 +437,14 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
             # broke 470 links. Two repositories and no tree to read: no link.
             selected: tuple[upstream.Repo, str] | None = None
             real_path = part.path
-            # A shipped binary (`DesktopKinect.dll`, `0x10037130-0x100371de`
-            # continuing a BAM.dll range) is no path of either repository:
-            # without a cached tree the single declared one took it.
-            if path_re.fullmatch(part.path) and not is_binary_citation(part.path):
+            if path_re.fullmatch(part.path):
                 if current_role:
                     # Only the repository the part names, or no link at all.
                     selected = by_role.get(current_role)
-                    if selected is not None and tree_holds(*selected, part.path) is False:
+                    if selected is not None and not tree_holds(*selected, part.path):
                         selected = None
                 else:
-                    selected = _holder(
-                        candidates, part.path,
-                        candidates[0] if len(candidates) == 1 else None,
-                    )
+                    selected = _holder(candidates, part.path)
 
             tag = (f" ({role_label(profile, current_role, part.path)})"
                    if current_role and _two_repositories(profile) else "")
@@ -344,6 +464,22 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
     return ", ".join(rendered_groups)
 
 
+def _binary_citation(path: str, previous: str) -> tuple[str, str]:
+    """A part citing code inside a shipped binary, and the binary it names.
+
+    `Nesoid 2.5.0 libnes.so 0x455a4-0x45694, 0x8094-0x80ac` cites two
+    ranges of one library: the bare address continues the binary before it.
+    No revision of a repository holds those bytes, so nothing is linked.
+    """
+    found = _HEX_ADDRESS_RE.search(path)
+    if found is None:
+        return f"`{path}`", path
+    binary = path[:found.start()].rstrip(" :") or previous
+    if not binary:
+        return f"`{path}`", ""
+    return f"offset `{path[found.start():]}` in `{binary}`", binary
+
+
 def link_prose_citations(profile: dict, text: str) -> str:
     """Prose file:line citations as pinned links, where the target is certain.
 
@@ -381,7 +517,7 @@ def link_prose_citations(profile: dict, text: str) -> str:
                 continue
             if any(lo <= path_span[0] < hi for lo, hi in guarded):
                 continue
-            if tree_holds(repo, pin, path) is False:
+            if not tree_holds(repo, pin, path):
                 continue
             for index, ((lo, hi), part) in enumerate(zip(spans, parts)):
                 start = path_span[0] if index == 0 else lo
@@ -494,3 +630,44 @@ def launcher_ref_markdown(source: ListSource | None, ref: str) -> str:
         else:
             rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})")
     return ", ".join(rendered)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        description="Record which cited source paths the cached upstream trees hold",
+    )
+    parser.add_argument(
+        "--cache-dir", type=Path, action="append",
+        help=f"tree listings profile_sync cached, read in the order given (default: {TREE_CACHE})",
+    )
+    parser.add_argument(
+        "--fetch", action="store_true",
+        help="fetch a tree no cache holds into the last --cache-dir",
+    )
+    parser.add_argument(
+        "--emulators-dir", default="emulators",
+        help="profiles whose citations are checked (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--output", type=Path, default=SOURCE_PATHS,
+        help="record the site reads (default: %(default)s)",
+    )
+    args = parser.parse_args()
+    cache_dirs = args.cache_dir or [TREE_CACHE]
+    if not args.fetch and not any(path.is_dir() for path in cache_dirs):
+        print(f"Error: no tree cache at {', '.join(map(str, cache_dirs))}", file=sys.stderr)
+        return 1
+    from common import load_emulator_profiles, write_if_changed  # noqa: PLC0415
+
+    profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
+    held, unread = build_source_paths(profiles, cache_dirs, args.fetch)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    write_if_changed(str(args.output), json.dumps(held, indent=1, sort_keys=True) + "\n")
+    print(f"{sum(map(len, held.values()))} paths held in {len(held)} trees, "
+          f"{len(unread)} trees not cached: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_site_exports.py

@@ -8,6 +8,7 @@ import sqlite3
 import sys
 import tempfile
 import unittest
+from unittest import mock
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -138,6 +139,7 @@ class SiteReferenceContracts(unittest.TestCase):
         }
         return db, coverages, profiles, stats
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_source_references_are_revision_pinned_links(self):
         _, _, profiles, _ = self._fixtures()
         rendered = _source_ref_markdown(
@@ -168,6 +170,7 @@ class SiteReferenceContracts(unittest.TestCase):
         prefixed = _source_ref_markdown(profile, "example/jg.c:213-214")
         self.assertEqual(prefixed, "`example/jg.c:213-214`")
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_single_repository_links_the_path_as_written(self):
         profile = {"source": "https://github.com/PCSX2/pcsx2", "source_commit": "c" * 40}
         rendered = _source_ref_markdown(profile, "pcsx2/ImGui/FullscreenUI.cpp:755-759")
@@ -588,6 +591,7 @@ class OnlyAPinnedProfileClaimsAPin(unittest.TestCase):
 class ADataDirectorySaysWhatItIs(unittest.TestCase):
     """The page said `dolphin-sys` > `Sys` and nothing of the tree it packs."""
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_description_origin_mode_and_source(self):
         from generate_site import generate_emulator_page  # noqa: PLC0415
 
@@ -771,6 +775,7 @@ class ProseCitationsLinkToThePin(unittest.TestCase):
                "source_commit": "76c7a84f1f7e80f3e6f2bba96fe100cb24e99124",
                "files": [{"name": "x", "source_ref": "libretro.c:10"}]}
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_rooted_or_structured_path_is_linked(self):
         from sitesources import link_prose_citations
 
@@ -797,6 +802,7 @@ class ACitationSaysWhoseCodeItIs(unittest.TestCase):
                "upstream": "https://github.com/alekmaul/a5200",
                "source_commit": "a" * 40, "upstream_commit": "b" * 40}
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_each_part_links_to_its_repository_with_its_label(self):
         from sitesources import _source_ref_markdown, requested_by
 

tests/test_site_render.py

@@ -113,6 +113,7 @@ class StructuredBlocks(unittest.TestCase):
 
         return "\n".join(generate_site._render_structured_blocks(profile))
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_source_ref_in_a_block_is_a_pinned_link(self):
         """186 citations in analysis and firmware blocks were plain code."""
         page = self.render(self.profile("analysis", {"tmss": {
@@ -237,6 +238,7 @@ class ForgesUpstreamDoesNotRead(unittest.TestCase):
     """Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
     Heritage showed no revision, and their citations were plain code."""
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def links(self, url: str, ref: str, **extra: str) -> tuple[str, str]:
         import sitesources
 

tests/test_source_ref_targets.py

@@ -41,7 +41,7 @@ class BinaryContinuationsAreNotLinked(unittest.TestCase):
 
     def _render(self, value: str) -> str:
         original = sitesources._tree_paths
-        sitesources._tree_paths = lambda repo, pin: None  # type: ignore[assignment]
+        sitesources._tree_paths = lambda repo, pin: frozenset({"FPLoader.cpp"})  # type: ignore[assignment]
         try:
             return sitesources._source_ref_markdown(self.PROFILE, value)
         finally: