chore: merge audit fixes (site)

# Conflicts:
#	emulators/flycast.yml
#	emulators/primehack.yml

scripts/cross_reference.py

@@ -367,6 +367,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) -
     return any(actual[field] in values for field, values in declared.items())
 
 
+_tree_cache: dict[tuple[str, str], list[str]] = {}
+
+
+def _trees(data_root: str, bios_root: str) -> list[str]:
+    """Every directory under data/ and bios/, as a path with a trailing slash."""
+    key = (data_root, bios_root)
+    if key not in _tree_cache:
+        _tree_cache[key] = sorted(
+            f"{path.as_posix()}/"
+            for root in (data_root, bios_root) if Path(root).is_dir()
+            for path in Path(root).rglob("*")
+            if path.is_dir()
+            and not any(p.startswith(".") for p in path.relative_to(root).parts)
+        )
+    return _tree_cache[key]
+
+
+def held_copy(
+    entry: dict, data_root: str = "data", bios_root: str = "bios"
+) -> tuple[str, str] | None:
+    """The data/ file, directory or bios/ ZIP member holding an entry.
+
+    The database indexes the files under bios/, and an entry held otherwise
+    (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its
+    data directory) read as held with no word of where. Returns (path,
+    member), the member empty for a file or a directory. A file entry that
+    declares a hash is held only by a copy carrying it, as _data_hit judges.
+    """
+    names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])]
+    names = [str(n) for n in names if n]
+    if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]):
+        for name in names:
+            tail = "/" + name.strip("/") + "/"
+            for tree in _trees(data_root, bios_root):
+                if tree.endswith(tail):
+                    return tree, ""
+        return None
+    declared = _declared_content(entry)
+    locations = _supplemental_locations(data_root, bios_root)
+    for name in names:
+        for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()):
+            if not declared or _content_matches(location, declared):
+                return location
+    return None
+
+
 class FileConsumers:
     """Who reads each collected file, joined through the resolver.
 
@@ -469,6 +515,10 @@ class FileConsumers:
                     self.installed.setdefault(sha1, {}).setdefault(
                         platform, set()).add(str(entry.get("dest", "")))
 
+    def sha1_at(self, path: str) -> str | None:
+        """The file the database records at a collection path."""
+        return self._sha1_by_path.get(path)
+
     def path_of(self, sha1: str) -> str:
         """The collection path the database records for a file."""
         return str(self._db.get("files", {}).get(sha1, {}).get("path", ""))
@@ -757,17 +807,18 @@ def _cross_reference_profile(
             continue
 
         # --- resolve source provenance ---
-        storage = f.get("storage", "")
-        if storage in ("release", "large_file"):
+        # A release asset is held when the database records it, as any other
+        # file: storage says where the bytes are served from, not that the
+        # collection has them (kenji-nx's user-picked firmware package).
+        source = next(
+            (
+                found for version in versions[slot]
+                if (found := entry_source(version, index)) is not None
+            ),
+            "missing",
+        )
+        if source != "missing" and f.get("storage", "") in ("release", "large_file"):
             source = "large_file"
-        else:
-            source = next(
-                (
-                    found for version in versions[slot]
-                    if (found := entry_source(version, index)) is not None
-                ),
-                "missing",
-            )
 
         in_repo = source != "missing"
 

scripts/profile_sync.py

@@ -977,6 +977,9 @@ class ProfileReport:
     upstream_pin: str | None = None
     upstream_head: str | None = None
     upstream_origin: str | None = None
+    # The release tag the upstream view is pinned on, when it is one. That
+    # view is read at its tag, and a bump leaves upstream_commit there.
+    upstream_tag: str | None = None
     # Repositories whose pin is no ancestor of the branch the profile
     # tracks: history was rewritten, or the pin lies on another branch.
     orphaned: list[str] = field(default_factory=list)
@@ -1345,7 +1348,23 @@ def detect_pinned_tag(
     The declared `core_version` names the tag, which is looked up by name so
     that a repository publishing nightly tags cannot hide an old release behind
     pagination.
+
+    Only the repository of the documented build freezes the profile. An
+    `upstream` a port was cut from sits on a release tag as its base
+    (GroovyMAME on mame0289, triforce on Dolphin 2407) while the port itself
+    keeps moving: that view is judged against its tag by `tag_pinned_views`,
+    and the port is still compared with its HEAD and still bumped.
     """
+    documented = [view for view in views if view.field != "upstream"] or views
+    tagged = tag_pinned_views(profile, documented, cache_dir, offline)
+    return next(iter(tagged.values()), None)
+
+
+def tag_pinned_views(
+    profile: dict, views: list[RepoView], cache_dir: str, offline: bool
+) -> dict[tuple[str, str], str]:
+    """Views whose pin is the tag `core_version` names, keyed by repository."""
+    tagged: dict[tuple[str, str], str] = {}
     for view in views:
         if view.pin == view.head:
             continue
@@ -1353,8 +1372,9 @@ def detect_pinned_tag(
             profile.get("core_version"), view.repo.name
         ):
             if upstream.tag_commit(view.repo, tag, cache_dir, offline) == view.pin:
-                return tag
-    return None
+                tagged[(view.repo.host, view.repo.slug)] = tag
+                break
+    return tagged
 
 
 def _resolve_project_words(
@@ -1538,12 +1558,27 @@ def build_report(
     report.pinned_tag = frozen or detect_pinned_tag(
         profile, views, cache_dir, offline
     )
+    # A repository pinned on its release tag is read at that tag, whichever
+    # role it plays: comparing mame0174 refs with mamedev HEAD says nothing.
+    tagged = tag_pinned_views(profile, views, cache_dir, offline)
+    report.upstream_tag = next(
+        (
+            tagged[(view.repo.host, view.repo.slug)]
+            for view in views
+            if view.field == "upstream" and (view.repo.host, view.repo.slug) in tagged
+        ),
+        None,
+    )
+
+    def on_own_revision(view: RepoView) -> bool:
+        return view.pin == view.head or (view.repo.host, view.repo.slug) in tagged
+
     if not report.pinned_tag:
         # A tag or a held pin is off the tracked branch by design.
         report.orphaned = [
             f"{view.repo.slug}: pin {view.pin[:8]} is no ancestor of {view.head[:8]}"
             for view in views
-            if view.pin != view.head
+            if not on_own_revision(view)
             and upstream.compare(
                 view.repo, view.pin, view.head, cache_dir, offline
             ).diverged
@@ -1896,9 +1931,9 @@ def build_report(
         if report.pinned_tag:
             return True
         if not part.role and len(path_roles.get(part.path, ())) > 1:
-            return all(view.pin == view.head for view in views)
+            return all(on_own_revision(view) for view in views)
         view, _ = resolve_path(part.path, part.start, tuple(tokens), role_of(part))
-        return view.pin == view.head
+        return on_own_revision(view)
 
     def judge(
         part: RefPart, flag: bool, tokens: Sequence[str], hashes: Sequence[str]
@@ -1931,7 +1966,7 @@ def build_report(
                 # port's 305-356 read past the end of the upstream's 345
                 # lines. As in a comparison, a ref fails only everywhere.
                 for view in views:
-                    if view is owner or view.pin != view.head:
+                    if view is owner or not on_own_revision(view):
                         continue
                     lines = fetch_from(view.repo, view.pin, part.path)
                     if lines is None:
@@ -2048,6 +2083,11 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str:
         )
     elif report.pin and report.pin == report.head:
         lines.append("  pin is HEAD: checked for self-consistency")
+    if report.upstream_tag and not report.pinned_tag:
+        lines.append(
+            f"  upstream pinned to tag {report.upstream_tag}: checked against "
+            "its own revision, not against HEAD"
+        )
     for url in report.unread or []:
         lines.append(f"  not read (unsupported host): {url}")
     for line in report.orphaned:
@@ -2891,8 +2931,12 @@ def bump_commit(
             file=sys.stderr,
         )
         return False
+    # An upstream pinned on a release tag is the base the port was cut from:
+    # its HEAD is another program, so the bump leaves it on the tag.
     upstream_moves = bool(
-        report.upstream_head and report.upstream_pin != report.upstream_head
+        report.upstream_head
+        and report.upstream_pin != report.upstream_head
+        and not report.upstream_tag
     )
     if document.get("source_commit") == report.head and not upstream_moves:
         # Rewriting the pin to the value it already holds is not a bump, and

scripts/validate_schemas.py

@@ -419,7 +419,9 @@ def main() -> int:
             api_dir / name
             for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json")
             if (api_dir / name).exists()
-        ]
+        ] + sorted((api_dir / "platforms").glob("*.json")) + sorted(
+            (api_dir / "systems").glob("*.json")
+        )
         errors.extend(
             _validate_json_files(envelope_paths, "site-api-envelope.schema.json")
         )

scripts/verify.py

@@ -644,11 +644,10 @@ def find_undeclared_files(
                         entry["required"] = True
                     continue
 
-                # Resolution: storage flag, then name, then path basename
-                storage = f.get("storage", "")
-                if storage in ("release", "large_file"):
-                    in_repo = True
-                elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
+                # Resolution: name, then path basename
+                # storage: release names where the bytes are served from, not
+                # that the collection holds them: resolved like any file.
+                if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
                     # The entry states what its content should be and the builder
                     # drops a copy that contradicts it, so content decides here
                     # too. A name match would not do: generic names collide across