chore: merge core versions read from code

scripts/generate_site.py

@@ -2510,7 +2510,9 @@ def _emulator_description(markdown: str, key: str, title: str, intro: str) -> st
         row = {name: _plain_markdown(value) for name, value in re.findall(
             r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)}
         kind = row.get("Classification") or row.get("Type") or "emulator"
-        version = row.get("Version", "")
+        # The cell goes on to say where the version is read: the intro
+        # keeps the version alone.
+        version = row.get("Version", "").split(", ")[0]
         known = f", version {version}" if version and version[0].isdigit() else ""
         intro = f"{title}: {kind.lower()}{known}."
     elif title not in intro:
@@ -7757,7 +7759,8 @@ def emulator_romset(profile: dict) -> str:
     version = str(profile.get("core_version") or "")
     if profile.get("mame_version"):
         return f"MAME {profile['mame_version']}"
-    if "MAME" in emulator and _MAME_RELEASE.match(version):
+    # HBMAME's sets are its own homebrew sets, whatever MAME it is based on.
+    if "MAME" in emulator and "HBMAME" not in emulator and _MAME_RELEASE.match(version):
         return f"MAME {version}"
     if emulator.startswith("FinalBurn Neo") and version:
         return f"FinalBurn Neo {version}"
@@ -7766,6 +7769,114 @@ def emulator_romset(profile: dict) -> str:
     return ""
 
 
+def _version_said(value: object, basis: dict) -> str:
+    """One build's version and where the profile reads it."""
+    kind = basis.get("kind")
+    if kind == "none":
+        return "none stated in the source"
+    if value is None:
+        if kind == "unverified":
+            return f"unknown: {_UNVERIFIED.get(str(basis.get('why')), 'unread')}"
+        return "none published"
+    value = str(value)
+    if kind == "file":
+        return f"{value}, stated in `{basis.get('path')}`"
+    if kind == "tag":
+        distance = int(basis.get("distance") or 0)
+        if not distance:
+            return f"{value}, the tag at this revision"
+        return f"{value}, {distance} commit{'s' if distance != 1 else ''} after tag `{basis.get('tag')}`"
+    if kind == "package":
+        return f"{value}, read from {basis.get('artifact')}"
+    if kind == "unverified":
+        return f"{value}, not confirmed: {_UNVERIFIED.get(str(basis.get('why')), 'unread')}"
+    return value
+
+
+# Why a profile's version could not be read, said to a reader.
+_UNVERIFIED = {
+    "no_pin": "no revision recorded",
+    "host_unsupported": "its host cannot be read",
+    "host_gone": "its source is gone",
+    "host_unreachable": "its host did not answer",
+    "pin_off_branch": "its recorded revision is not the build described",
+    "no_artifact": "no build recorded as read",
+    "no_source": "no source declared",
+}
+
+
+def version_markdown(profile: dict) -> str:
+    """The Version cell: core_version and where version_basis reads it.
+
+    A null version said "none published" whether the code states none or
+    nobody looked. Keyed by build mode like source, each build is named,
+    where the page printed the dict's Python repr.
+    """
+    if "core_version" not in profile and not profile.get("version_basis"):
+        return "unknown"
+    value = profile.get("core_version")
+    basis = profile.get("version_basis") if isinstance(profile.get("version_basis"), dict) else {}
+    if isinstance(value, dict) or (basis and "kind" not in basis):
+        values = value if isinstance(value, dict) else {}
+        modes = list(dict.fromkeys([*values, *basis]))
+        return ", ".join(
+            f"{_version_said(values.get(mode), basis.get(mode) or {})} ({mode})"
+            for mode in modes
+        )
+    return _version_said(value, basis)
+
+
+ROMSET_FAQ_PAGE = Path(WIKI_SRC_DIR) / "technical-faq.md"
+_ROMSET_BLOCK = re.compile(
+    r"(<!-- generated: romsets -->\n)(.*?)(<!-- /generated: romsets -->)", re.S)
+
+
+def _retroarch_label(profile: dict) -> str:
+    """"Arcade (MAME 2003)" gives "MAME 2003", "Arcade (FinalBurn Neo)
+    (Neo-Geo)" gives "FinalBurn Neo (Neo-Geo)"."""
+    inner = str(profile.get("display_name") or profile.get("emulator") or "").split("(", 1)[-1]
+    return inner.replace(")", "", 1).strip()
+
+
+def romset_table(profiles: dict) -> list[str]:
+    """The FAQ's arcade table: each MAME and FinalBurn libretro core and the
+    romset its profile reads at its pin.
+
+    Written by hand, it told readers to collect MAME 0.160 sets for a core
+    whose src/version.c states 0.159, and FinalBurn Neo 1.0.0.2 for a source
+    at 1.0.0.03. A core whose name gives its version (Recalbox's MAME 0.258)
+    needs no row.
+    """
+    rows: dict[str, list[str]] = {}
+    for key, profile in sorted(profiles.items()):
+        if not re.match(r"(mame|fbneo|fbalpha)", key) or \
+                "libretro" not in str(profile.get("type", "")):
+            continue
+        romset = emulator_romset(profile)
+        label = _retroarch_label(profile)
+        if not romset or romset.split(" ", 1)[1].lstrip("v") in label:
+            continue
+        rows.setdefault(romset, []).append(label)
+
+    def order(romset: str) -> tuple:
+        family = 0 if romset.startswith("MAME") else 1 if romset.startswith("FB Alpha") else 2
+        numbers = tuple(int(n) for n in re.findall(r"\d+", romset.split(" (", 1)[0]))
+        return family, numbers
+
+    lines = ["| Emulator, as RetroArch names it | Sets from |", "|---|---|"]
+    for romset in sorted(rows, key=order):
+        lines.append(f"| {', '.join(sorted(rows[romset]))} | {romset} |")
+    return lines
+
+
+def with_romset_table(page: str, profiles: dict) -> str:
+    """The FAQ page with its romset table read from the profiles."""
+    table = "\n".join(romset_table(profiles)) + "\n"
+    if not _ROMSET_BLOCK.search(page):
+        raise ValueError(f"{ROMSET_FAQ_PAGE}: no romsets block to write the table in")
+    return _ROMSET_BLOCK.sub(lambda m: f"{m.group(1)}{table}{m.group(3)}", page, count=1)
+
+
 def _render_metadata_rows(profile: dict) -> list[str]:
     """Scalar profile fields, one table row each.
 
@@ -8856,15 +8967,7 @@ def emulator_pages(
     upstream_raw = profile.get("upstream", "")
     upstream_none = "upstream" in profile and upstream_raw is None
     upstream = str(upstream_raw) if isinstance(upstream_raw, str) else ""
-    version_raw = profile.get("core_version", "unknown")
-    if version_raw is None:
-        version = "none published"
-    elif isinstance(version_raw, dict):
-        # Keyed by build mode like source: each build's version, named,
-        # where the page printed the dict's Python repr.
-        version = ", ".join(f"{v} ({mode})" for mode, v in version_raw.items() if v)
-    else:
-        version = version_raw
+    version = version_markdown(profile)
     profiled = profile.get("profiled_date", "unknown")
     systems = profile.get("systems", [])
     cores = profile.get("cores", [name])
@@ -11334,6 +11437,11 @@ def _generate(args: argparse.Namespace) -> None:
     wiki_dest = docs / "wiki"
     wiki_dest.mkdir(parents=True, exist_ok=True)
     wiki_src = Path(WIKI_SRC_DIR)
+    # The romset table is the profiles' versions, never typed: the source
+    # page is rewritten before it is copied, as the README is.
+    faq = _REPO_ROOT / ROMSET_FAQ_PAGE
+    if faq.is_file():
+        write_if_changed(str(faq), with_romset_table(faq.read_text(encoding="utf-8"), profiles))
     if wiki_src.is_dir():
         for src_file in wiki_src.glob("*.md"):
             shutil.copy2(src_file, wiki_dest / src_file.name)

scripts/profile_sync.py

@@ -31,6 +31,7 @@ import yaml
 sys.path.insert(0, os.path.dirname(__file__))
 
 import upstream
+import version_basis
 from common import (
     _norm_system_id,
     github_token,
@@ -2670,6 +2671,129 @@ def apply_edit(path: Path, new_text: str, expected: dict) -> None:
         raise YamlWriteError(f"{path}: structure changed beyond the intended field")
 
 
+# Key order of a written version_basis, as a reader expects it.
+_BASIS_ORDER = ("kind", "why", "repo", "url", "commit", "path", "pattern", "format",
+                "match", "tag", "distance", "of", "artifact")
+
+
+def _top_block(lines: list[str], key: str) -> tuple[int, int] | None:
+    """Line span of a top-level key and the indented lines under it."""
+    indices = _top_level_indices(lines, key)
+    if not indices:
+        return None
+    if len(indices) > 1:
+        raise YamlWriteError(f"{key}: expected one top-level line, found {len(indices)}")
+    end = indices[0] + 1
+    while end < len(lines) and (lines[end].startswith((" ", "\t")) or not lines[end].strip()):
+        end += 1
+    # Blank lines closing the block belong to the document, not to it.
+    while end > indices[0] + 1 and not lines[end - 1].strip():
+        end -= 1
+    return indices[0], end
+
+
+def _render_version(value: object) -> list[str]:
+    if isinstance(value, dict):
+        return ["core_version:"] + [
+            f"  {mode}: {'null' if v is None else _double_quoted(str(v))}"
+            for mode, v in value.items()
+        ]
+    return [f"core_version: {'null' if value is None else _double_quoted(str(value))}"]
+
+
+def _render_basis(basis: dict) -> list[str]:
+    def ordered(entry: dict) -> dict:
+        rank = {key: i for i, key in enumerate(_BASIS_ORDER)}
+        return dict(sorted(entry.items(), key=lambda item: rank.get(item[0], len(rank))))
+
+    def scalar(value: object) -> str:
+        # Every string quoted: a tag 2407 or 0.10 read back as a number. A
+        # pattern keeps its backslashes readable in single quotes.
+        if isinstance(value, bool) or not isinstance(value, str):
+            return str(value).lower() if isinstance(value, bool) else str(value)
+        if "\\" in value or '"' in value:
+            return "'" + value.replace("'", "''") + "'"
+        return _double_quoted(value)
+
+    def block(entry: dict, indent: str) -> list[str]:
+        # kind, why and repo are words of the schema's enums, written bare.
+        return [f"{indent}{key}: {value if key in ('kind', 'why', 'repo') else scalar(value)}"
+                for key, value in ordered(entry).items()]
+
+    if "kind" in basis:
+        return ["version_basis:", *block(basis, "  ")]
+    lines = ["version_basis:"]
+    for mode, entry in basis.items():
+        lines += [f"  {mode}:", *block(entry, "    ")]
+    return lines
+
+
+def set_version_fields(text: str, version: object, basis: dict | None) -> str:
+    """Rewrite core_version and version_basis, version_basis right after it."""
+    lines = text.splitlines()
+    span = _top_block(lines, "core_version")
+    rendered = _render_version(version)
+    if span is None:
+        anchor = _top_block(lines, "upstream_commit") or _top_block(lines, "source_commit") \
+            or _top_block(lines, "profiled_date") or _top_block(lines, "upstream") \
+            or _top_block(lines, "source") or _top_block(lines, "alias_of") \
+            or _top_block(lines, "type")
+        if anchor is None:
+            raise YamlWriteError("core_version: no line to write it after")
+        lines[anchor[1]:anchor[1]] = rendered
+    else:
+        lines[span[0]:span[1]] = rendered
+    if basis is not None:
+        old = _top_block(lines, "version_basis")
+        if old is not None:
+            del lines[old[0]:old[1]]
+        after = _top_block(lines, "core_version")
+        lines[after[1]:after[1]] = _render_basis(basis)
+    return "\n".join(lines) + ("\n" if text.endswith("\n") else "")
+
+
+def apply_readings(document: dict, readings: list) -> tuple[object, dict | None]:
+    """core_version and version_basis as the readings state them.
+
+    A reading that could not be made leaves its build mode as written.
+    """
+    version = document.get("core_version")
+    basis = document.get("version_basis")
+    basis = dict(basis) if isinstance(basis, dict) else None
+    if isinstance(version, dict):
+        version = dict(version)
+    for reading in readings:
+        if reading.status not in (version_basis.MATCH, version_basis.DRIFT):
+            continue
+        if reading.mode:
+            if not isinstance(version, dict):
+                version = {}
+            version[reading.mode] = reading.read
+        else:
+            version = reading.read
+        if reading.updates and basis is not None:
+            target = basis if not reading.mode else dict(basis.get(reading.mode) or {})
+            target.update(dict(reading.updates))
+            if reading.mode:
+                basis[reading.mode] = target
+    return version, basis
+
+
+def write_version(path: Path, readings: list) -> bool:
+    """Set core_version (and a tag basis' tag and distance) from the readings."""
+    text = path.read_text(encoding="utf-8")
+    document = yaml.safe_load(text)
+    version, basis = apply_readings(document, readings)
+    if version == document.get("core_version") and basis == document.get("version_basis"):
+        return False
+    expected = dict(document)
+    expected["core_version"] = version
+    if basis is not None:
+        expected["version_basis"] = basis
+    apply_edit(path, set_version_fields(text, version, basis), expected)
+    return True
+
+
 def backfill_commit(path: Path, sha: str) -> bool:
     """Insert source_commit after profiled_date. False when already present."""
     text = path.read_text(encoding="utf-8")
@@ -3313,9 +3437,15 @@ def _needs_review(report: ProfileReport, accept_changed: bool) -> bool:
 
 
 def bump_commit(
-    path: Path, report: ProfileReport, accept_changed: bool = False
+    path: Path, report: ProfileReport, accept_changed: bool = False,
+    cache_dir: str = DEFAULT_CACHE, offline: bool = False,
 ) -> bool:
-    """Advance source_commit to HEAD when nothing needs a read again."""
+    """Advance source_commit to HEAD when nothing needs a read again.
+
+    core_version moves with the pin, read again from version_basis at the
+    new revision. Four bulk advances moved 295 pins and no version: an
+    advance whose version cannot be read is refused.
+    """
     if _needs_review(report, accept_changed):
         return False
     text = path.read_text(encoding="utf-8")
@@ -3375,10 +3505,33 @@ def bump_commit(
         new_text = _set_pin(
             new_text, document, "upstream_commit", report.upstream_head, "source_commit"
         )
+    readings = version_basis.check(expected, cache_dir, offline, recompute=True)
+    refused = version_refusal(readings)
+    if refused:
+        print(f"{path.stem}: {refused}, its pin is not advanced", file=sys.stderr)
+        return False
+    version, basis = apply_readings(expected, readings)
+    if version != expected.get("core_version") or basis != expected.get("version_basis"):
+        new_text = set_version_fields(new_text, version, basis)
+        expected["core_version"] = version
+        if basis is not None:
+            expected["version_basis"] = basis
     apply_edit(path, new_text, expected)
     return True
 
 
+def version_refusal(readings: list) -> str | None:
+    """Why the version at a new pin cannot be stated, or None."""
+    for reading in readings:
+        if reading.status == version_basis.NO_BASIS:
+            return "no version_basis says where its version is read"
+        if reading.status == version_basis.UNREADABLE:
+            return f"core_version cannot be read at the new pin: {reading.where}"
+        if reading.status == version_basis.UNCHECKED:
+            return f"core_version is {reading.where}, not read from the code at a pin"
+    return None
+
+
 def pins_changed(before: dict, after: dict) -> list[str]:
     """The pin fields whose value a write changed."""
     return [
@@ -3996,6 +4149,16 @@ def build_parser() -> argparse.ArgumentParser:
         "--check-version", action="store_true",
         help="compare core_version with the latest tag and release",
     )
+    parser.add_argument(
+        "--verify-version", action="store_true",
+        help="read core_version again from version_basis at the pin and "
+        "exit 1 when the code states another",
+    )
+    parser.add_argument(
+        "--set-version", action="store_true",
+        help="write core_version from version_basis at the pin, the nearest "
+        "tag asked of git again",
+    )
     parser.add_argument(
         "--detect-new-files", action="store_true",
         help="name files upstream loads that the profile does not list",
@@ -4284,7 +4447,8 @@ def _apply_writes(
         applied = []
         if args.rebase_refs and not report.skipped:
             applied = rebase_refs(target, report, args.accept_changed)
-        moved = bump_commit(target, report, args.accept_changed)
+        moved = bump_commit(target, report, args.accept_changed,
+                            args.cache_dir, args.offline)
         if args.rebase_refs and applied and not moved:
             keyed = any(isinstance(profile.get(f), dict) for f in ("source_commit", "upstream_commit"))
             reason = (
@@ -4723,6 +4887,32 @@ def _print_triage(
 
 def _refuse_flag_combinations(args: argparse.Namespace) -> None:
     """Exit on a flag the chosen mode would not apply, before any request."""
+    if args.verify_version or args.set_version:
+        ignored = [
+            flag for flag, on in (
+                ("--verify-version", args.verify_version and args.set_version),
+                ("--markdown", args.markdown),
+                ("--fetch-plan", args.fetch_plan),
+                ("--triage", args.triage),
+                ("--strict", args.strict),
+                ("--check-version", args.check_version),
+                ("--detect-new-files", args.detect_new_files),
+                ("--watch-hashes", args.watch_hashes),
+                ("--full-diff", args.full_diff),
+                ("--tree-diff", args.tree_diff),
+                ("--backfill-commits", args.backfill_commits),
+                ("--rebase-refs", args.rebase_refs),
+                ("--attribute-refs", args.attribute_refs),
+                ("--bump-commit", args.bump_commit),
+                ("--realign-prose", args.realign_prose),
+                ("--accept-changed", args.accept_changed),
+                ("--dry-run", args.dry_run and not args.set_version),
+            ) if on
+        ]
+        if ignored:
+            mode = "--set-version" if args.set_version else "--verify-version"
+            print(f"{mode} does not apply {', '.join(ignored)}", file=sys.stderr)
+            raise SystemExit(1)
     if args.fresh and args.offline:
         print("--fresh asks the forges again, which --offline forbids.", file=sys.stderr)
         raise SystemExit(1)
@@ -4809,6 +4999,63 @@ def _refuse_flag_combinations(args: argparse.Namespace) -> None:
         raise SystemExit(1)
 
 
+def _version_selection(args: argparse.Namespace, profiles: dict) -> dict:
+    """--all reaches the alias and test profiles too: each states a version."""
+    if args.all and not (args.system or args.type_ or args.stale_before or args.limit):
+        return dict(sorted(profiles.items()))
+    return select_profiles(profiles, args)
+
+
+def format_reading(name: str, reading: version_basis.Reading) -> str:
+    mode = f"[{reading.mode}] " if reading.mode else ""
+    kind = f"{reading.kind} " if reading.kind else ""
+    if reading.status == version_basis.DRIFT:
+        said = f"declared {reading.declared}, code {reading.read}"
+    elif reading.status == version_basis.MATCH:
+        said = str(reading.read)
+    else:
+        said = f"declared {reading.declared}"
+    return f"{name}: {mode}{reading.status} {said} ({kind}{reading.where})"
+
+
+def run_versions(args: argparse.Namespace, profiles: dict) -> None:
+    """--verify-version and --set-version: core_version against the code."""
+    selected = _version_selection(args, profiles)
+    if args.set_version and not args.dry_run and not args.force:
+        _refuse_dirty_tree(args.emulators_dir, list(selected))
+    counts: Counter = Counter()
+    rows = []
+    failed = False
+    for name, profile in selected.items():
+        readings = version_basis.check(
+            profile, args.cache_dir, args.offline, profiles, recompute=args.set_version
+        )
+        for reading in readings:
+            counts[reading.status] += 1
+            rows.append({"profile": name, **asdict(reading)})
+            if not args.as_json and (not args.changed_only or reading.status != version_basis.MATCH):
+                print(format_reading(name, reading))
+        if args.set_version:
+            path = Path(args.emulators_dir) / f"{name}.yml"
+            if args.dry_run:
+                continue
+            try:
+                if write_version(path, readings):
+                    print(f"{name}: core_version written")
+            except YamlWriteError as exc:
+                print(f"{name}: write refused: {exc}", file=sys.stderr)
+                failed = True
+        elif any(r.status in version_basis.FAILING for r in readings):
+            failed = True
+    if args.as_json:
+        print(json.dumps(rows, indent=2))
+    else:
+        print(", ".join(f"{count} {status}" for status, count in sorted(counts.items())),
+              file=sys.stderr)
+    if failed:
+        raise SystemExit(1)
+
+
 def main() -> None:
     """Build one report per selected profile, then apply writes and output."""
     args = build_parser().parse_args()
@@ -4818,6 +5065,9 @@ def main() -> None:
         # trzy/Supermodel and VICE read as behind their real HEAD.
         upstream.MOVING_TTL = 0
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
+    if args.verify_version or args.set_version:
+        run_versions(args, profiles)
+        return
     selected = select_profiles(profiles, args)
     _check_quota(len(selected), args.offline)
 

scripts/scraper/_hash_merge.py

@@ -158,7 +158,12 @@ def merge_fbneo_profile(
     profile = _load_yaml(profile_path)
     hashes = _load_json(hashes_path)
 
-    profile["core_version"] = hashes.get("version", profile.get("core_version"))
+    # core_version stays as written: it is the version of the code at the
+    # profile's own pin, read through version_basis (profile_sync
+    # --set-version). The hashes come from finalburnneo/FBNeo's tip, a
+    # revision the libretro/FBNeo pin does not name: writing its version
+    # left fbneo, fbneo_cps12 and fbneo_neogeo on a release their pin
+    # never was.
 
     files = profile.get("files", [])
     archive_files, non_archive = _split_files(files, lambda f: "archive" in f)
@@ -1060,17 +1065,16 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
     """Write merged FBNeo profile using text-based patching.
 
     FBNeo profiles have individual ROM entries with archive: field.
-    Patches core_version, the content values of existing entries that
-    changed, and appends new ROM entries. Advancing core_version alone left
-    a profile claiming a release whose crc32 it did not carry. Comments and
-    curated source_refs stay as written.
+    Patches the content values of existing entries that changed and
+    appends new ROM entries. core_version is the pin's, never written here.
+    Comments and curated source_refs stay as written.
     """
     p = Path(path)
     backup = p.with_suffix(PROFILE_BACKUP_SUFFIX)
     shutil.copy2(p, backup)
 
     original = p.read_text(encoding="utf-8")
-    patched = _patch_core_version(original, data.get("core_version", ""))
+    patched = original
 
     # Identify new ROM entries by comparing parsed data keys, not text search
     existing_data = yaml_load(original) or {}

scripts/scraper/fbneo_hash_scraper.py

@@ -11,6 +11,7 @@ import argparse
 import json
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -20,7 +21,7 @@ from typing import Any
 
 import yaml
 
-from common import github_token, source_yaml_paths, write_text_atomic, yaml_load
+from common import source_yaml_paths, write_text_atomic, yaml_load
 
 from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile
 from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
@@ -51,7 +52,7 @@ def _is_cache_fresh() -> bool:
 
 
 def _sparse_clone() -> None:
-    """Sparse clone FBNeo repo, checking out only src/burn/drv."""
+    """Sparse clone FBNeo repo: src/burn/drv, and in cone mode src/burn/version.h."""
     if CLONE_DIR.exists():
         shutil.rmtree(CLONE_DIR)
 
@@ -74,7 +75,7 @@ def _sparse_clone() -> None:
     )
 
     subprocess.run(
-        ["git", "sparse-checkout", "set", "src/burn/drv", "src/burner/resource.h"],
+        ["git", "sparse-checkout", "set", "src/burn/drv"],
         cwd=CLONE_DIR,
         check=True,
         capture_output=True,
@@ -82,31 +83,28 @@ def _sparse_clone() -> None:
     )
 
 
+# src/burn/version.h, as the libretro core prints it: "v%x.%x.%x.%02x" of
+# the BCD BURN_VERSION, so VER_ALPHA 3 reads 1.0.0.03. FBNeo bumps this file
+# without tagging: the last tag (v1.0.0.02) was 9 337 commits behind it.
+_VERSION_H = re.compile(
+    r"#define\s+VER_MAJOR\s+(\d+)\s+#define\s+VER_MINOR\s+(\d+)\s+"
+    r"#define\s+VER_BETA\s+(\d+)\s+#define\s+VER_ALPHA\s+(\d+)"
+)
+
+
+def version_from_header(text: str) -> str:
+    """The version src/burn/version.h states."""
+    found = _VERSION_H.search(text)
+    if not found:
+        raise RuntimeError("src/burn/version.h states no VER_MAJOR..VER_ALPHA")
+    major, minor, beta, alpha = (int(group) for group in found.groups())
+    return f"v{major}.{minor}.{beta}.{alpha:02d}"
+
+
 def _extract_version() -> tuple[str, str]:
-    """Extract version tag and commit SHA from the cloned repo.
-
-    Returns (version, commit_sha). Falls back to resource.h if no tag.
-    """
-    result = subprocess.run(
-        ["git", "describe", "--tags", "--abbrev=0"],
-        cwd=CLONE_DIR,
-        capture_output=True,
-        text=True,
-    )
-
-    # Prefer real version tags over pseudo-tags like "latest"
-    version = "unknown"
-    if result.returncode == 0:
-        tag = result.stdout.strip()
-        if tag and tag != "latest":
-            version = tag
-    # Fallback: resource.h
-    if version == "unknown":
-        version = _version_from_resource_h()
-    # Last resort: use GitHub API for latest real release tag
-    if version == "unknown":
-        version = _version_from_api()
-
+    """The version the cloned source states, and its commit."""
+    header = CLONE_DIR / "src" / "burn" / "version.h"
+    version = version_from_header(header.read_text(encoding="utf-8", errors="replace"))
     sha_result = subprocess.run(
         ["git", "rev-parse", "HEAD"],
         cwd=CLONE_DIR,
@@ -114,49 +112,7 @@ def _extract_version() -> tuple[str, str]:
         text=True,
         check=True,
     )
-    commit = sha_result.stdout.strip()
-
-    return version, commit
-
-
-def _version_from_api() -> str:
-    """The latest real release tag, or an error: "unknown" was written as
-    core_version and cached for a day."""
-    import json as json_mod
-    import urllib.error
-    import urllib.request
-
-    headers = {"User-Agent": "retrobios-scraper/1.0"}
-    token = github_token()
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
-    req = urllib.request.Request(
-        "https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10", headers=headers
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=10) as resp:
-            tags = json_mod.loads(resp.read())
-    except (urllib.error.URLError, OSError, ValueError) as exc:
-        raise RuntimeError(f"cannot determine the FBNeo version: {exc}") from exc
-    for tag in tags:
-        if tag["name"] != "latest" and tag["name"].startswith("v"):
-            return tag["name"]
-    raise RuntimeError("cannot determine the FBNeo version: no tag, resource.h or release")
-
-
-def _version_from_resource_h() -> str:
-    """Fallback: parse VER_FULL_VERSION_STR from resource.h."""
-    resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
-    if not resource_h.exists():
-        return "unknown"
-
-    text = resource_h.read_text(encoding="utf-8", errors="replace")
-    for line in text.splitlines():
-        if "VER_FULL_VERSION_STR" in line:
-            parts = line.split('"')
-            if len(parts) >= 2:
-                return parts[1]
-    return "unknown"
+    return version, sha_result.stdout.strip()
 
 
 def _cleanup() -> None:

scripts/scraper/mame_hash_scraper.py

@@ -467,6 +467,13 @@ def release_profile(
         "source_commit": source_commit,
         "upstream_commit": upstream_commit,
         "core_version": version,
+        # Where profile_sync reads it again: the makefile of the libretro/mame
+        # tree the platform builds, at source_commit.
+        "version_basis": {
+            "kind": "file",
+            "path": "makefile",
+            "pattern": '#define BARE_BUILD_VERSION "([^"]+)"',
+        },
         "display_name": f"Arcade (MAME {version})",
         "cores": cores,
         "built_by": built_by,

scripts/upstream.py

@@ -12,6 +12,7 @@ import http.client
 import json
 import os
 import socket
+import subprocess
 import tempfile
 import time
 import urllib.error
@@ -1086,3 +1087,103 @@ def commits_touching(
         1 for entry in payload
         if isinstance(entry, dict) and (entry.get("sha") or entry.get("id")) != base
     )
+
+
+# Commit graphs kept to count the commits between a tag and a pin. A
+# treeless bare clone holds commits and tags only: the files a profile reads
+# come from fetch_file, by sha.
+GIT_DIR = "_git"
+
+
+def clone_dir(cache_dir: str, repo: Repo) -> Path:
+    """Where the commit graph of a repository is kept."""
+    return Path(cache_dir) / GIT_DIR / repo.host / repo.owner / f"{repo.name}.git"
+
+
+def _git(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess:
+    env = dict(os.environ, GIT_TERMINAL_PROMPT="0")
+    return subprocess.run(
+        ["git", *args], cwd=cwd, capture_output=True, text=True, env=env, check=False
+    )
+
+
+def _has_commit(clone: Path, sha: str) -> bool:
+    return _git(["cat-file", "-e", f"{sha}^{{commit}}"], clone).returncode == 0
+
+
+def _graph(repo: Repo, sha: str, cache_dir: str, offline: bool) -> Path:
+    """A clone holding sha and every tag, fetched again once tags may have moved."""
+    clone = clone_dir(cache_dir, repo)
+    url = f"https://{repo.host}/{repo.slug}.git"
+    if not (clone / "HEAD").is_file():
+        if offline:
+            raise NotCachedError(f"{repo.slug}: no commit graph in the offline cache")
+        clone.parent.mkdir(parents=True, exist_ok=True)
+        result = _git(["clone", "--quiet", "--bare", "--filter=tree:0", url, str(clone)])
+        if result.returncode:
+            raise UpstreamError(f"{url}: clone failed: {result.stderr.strip()}")
+        _git(["fetch", "--quiet", "--filter=tree:0", "--tags", "origin"], clone)
+    stamp = clone / "FETCH_HEAD"
+    fresh = stamp.is_file() and time.time() - stamp.stat().st_mtime <= MOVING_TTL
+    if offline or (fresh and _has_commit(clone, sha)):
+        if not _has_commit(clone, sha):
+            raise NotCachedError(f"{repo.slug}@{sha[:8]}: not in the offline commit graph")
+        return clone
+    # Tags are fetched again: a tag pushed since the clone can be nearer.
+    result = _git(
+        ["fetch", "--quiet", "--filter=tree:0", "--force", "--tags", "origin",
+         "+refs/heads/*:refs/heads/*"], clone,
+    )
+    if result.returncode:
+        raise UpstreamError(f"{url}: fetch failed: {result.stderr.strip()}")
+    if not _has_commit(clone, sha):
+        # A commit no branch reaches any more: a rewritten fork, a PR head.
+        _git(["fetch", "--quiet", "--filter=tree:0", "origin", sha], clone)
+    if not _has_commit(clone, sha):
+        raise UpstreamError(f"{url}: {sha[:8]} is on no branch the forge serves")
+    return clone
+
+
+def describe(
+    repo: Repo, sha: str, cache_dir: str, offline: bool = False,
+    match: str | None = None,
+) -> tuple[str, int] | None:
+    """The tag nearest to sha in its history, and the commits after it.
+
+    git describe counts the commits sha has that the tag lacks, the count a
+    forge compare reports as ahead. ``match`` is a glob the tag must fit.
+    None when no tag is reachable from sha.
+    """
+    clone = _graph(repo, sha, cache_dir, offline)
+    argv = ["describe", "--tags", "--long", "--abbrev=40"]
+    if match:
+        argv += ["--match", match]
+    result = _git([*argv, sha], clone)
+    if result.returncode:
+        if any(text in result.stderr for text in (
+            "No names found", "No tags can describe", "cannot describe",
+        )):
+            return None
+        raise UpstreamError(f"{repo.slug}: describe {sha[:8]}: {result.stderr.strip()}")
+    tag, _distance, _commit = result.stdout.strip().rsplit("-", 2)
+    # describe's own count is an estimate across merges (ares: 149 where the
+    # pin holds 143 commits the tag lacks): the exact count is rev-list's.
+    counted = _git(["rev-list", "--count", f"refs/tags/{tag}..{sha}"], clone)
+    if counted.returncode:
+        raise UpstreamError(f"{repo.slug}: {tag}..{sha[:8]}: {counted.stderr.strip()}")
+    return tag, int(counted.stdout.strip())
+
+
+def graph_ancestry(
+    repo: Repo, base: str, head: str, cache_dir: str, offline: bool = False
+) -> tuple[int, int]:
+    """(ahead, behind) counted in the commit graph, for a pair a forge
+    refuses to compare (GitHub answers 422 past a diff size)."""
+    clone = _graph(repo, head, cache_dir, offline)
+    counts = []
+    for spec in (f"{base}..{head}", f"{head}..{base}"):
+        result = _git(["rev-list", "--count", spec], clone)
+        if result.returncode:
+            raise UpstreamError(f"{repo.slug}: {spec}: {result.stderr.strip()}")
+        counts.append(int(result.stdout.strip()))
+    return counts[0], counts[1]

scripts/version_basis.py

@@ -0,0 +1,280 @@
+"""Read a profile's core_version from the source its version_basis records.
+
+A profile states the version of the code it was read at. Copied from a
+libretro .info or left behind by a pin advance, it named another release:
+snes9x said 1.61 at a pin whose snes9x.h states 1.63, and four bulk advances
+moved 295 pins without touching one version. version_basis records where the
+version is read, so a tool can read it again at any pin:
+
+- ``file``: a pattern in one file of a declared repository at its pin, the
+  version the program states (a macro, a build file, a package file).
+- ``tag``: the nearest tag in the pin's history and the commits after it,
+  for code that takes its version from git at build time. core_version is
+  the tag at distance 0, else ``<tag>-<distance>-g<pin7>``, git describe's
+  form.
+- ``alias``: the same binary as another profile, whose version it carries.
+- ``package``: an artefact read outside any source tree (an Android package,
+  a closed binary, a release archive): a version no pin can be asked for.
+  A pattern reads it from the artefact's name when the name states it.
+- ``none``: the code at the pin states no version and no tag reaches it.
+  core_version is then null.
+- ``unverified``: no reading could be made (``why``: no pin, a host the
+  tools cannot read, a forge gone, no build named as read, no source).
+  core_version stays as written and is reported unconfirmed.
+
+A profile whose builds live in different repositories keys version_basis by
+build mode, the way it keys core_version.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import upstream
+
+KINDS = ("file", "tag", "alias", "package", "none", "unverified")
+# Values copied from a .info or a branch name, which name no version.
+PLACEHOLDERS = frozenset({"git", "svn", "unknown", "master", "main", "latest", "dev", "head"})
+# Pin field of each repository field a basis may name. A mirror carries the
+# source's history, so source_commit pins it too.
+PIN_FIELDS = {"source": "source_commit", "source_mirror": "source_commit",
+              "upstream": "upstream_commit"}
+
+MATCH = "MATCH"
+DRIFT = "DRIFT"
+UNREADABLE = "UNREADABLE"
+NO_BASIS = "NO_BASIS"
+UNCHECKED = "UNCHECKED"
+FAILING = (DRIFT, UNREADABLE, NO_BASIS)
+
+_FIELD = re.compile(r"\{(\d+)(?::([^}]*))?\}")
+
+
+class VersionUnreadable(Exception):
+    """The basis yields no version at the pin it is asked about."""
+
+
+@dataclass(frozen=True)
+class Reading:
+    mode: str
+    kind: str
+    declared: str | None
+    read: str | None
+    where: str
+    status: str
+    # What a write sets beside core_version: the tag and distance of a tag basis.
+    updates: tuple[tuple[str, object], ...] = ()
+
+
+def bases(profile: dict) -> dict[str, dict]:
+    """The basis of each build mode, "" when one covers every build."""
+    basis = profile.get("version_basis")
+    if not isinstance(basis, dict):
+        return {}
+    if "kind" in basis:
+        return {"": basis}
+    return {str(mode): value for mode, value in basis.items() if isinstance(value, dict)}
+
+
+def declared(profile: dict, mode: str) -> str | None:
+    """core_version for a build mode, as written."""
+    value = profile.get("core_version")
+    if isinstance(value, dict):
+        value = value.get(mode)
+    return None if value is None else str(value)
+
+
+def _by_mode(profile: dict, field: str, mode: str) -> str | None:
+    value = profile.get(field)
+    if isinstance(value, dict):
+        value = value.get(mode)
+    return str(value) if value else None
+
+
+def is_placeholder(value: object) -> bool:
+    """A VCS or branch word standing for a version, alone or annotated
+    ("SVN (2015 snapshot)"). The schema refuses the same strings."""
+    if not isinstance(value, str):
+        return False
+    return re.split(r"[\s(]", value.strip(), maxsplit=1)[0].casefold() in PLACEHOLDERS
+
+
+def format_version(match: re.Match, template: str | None) -> str:
+    """The groups of a match laid out by template, `{1}.{2}` or `{4:02d}`.
+
+    A format spec applies to the group read as an integer: FBNeo writes its
+    alpha number on two digits (VER_ALPHA 3 is 1.0.0.03).
+    """
+    if not template:
+        return match.group(1).strip()
+
+    def field(found: re.Match) -> str:
+        text = match.group(int(found.group(1))) or ""
+        spec = found.group(2)
+        return format(int(text, 0), spec) if spec else text.strip()
+
+    return _FIELD.sub(field, template)
+
+
+def read_text_version(text: str, pattern: str, template: str | None) -> tuple[str, int] | None:
+    """The version a pattern reads in a file's text, and its line number."""
+    found = re.search(pattern, text, re.MULTILINE)
+    if not found:
+        return None
+    return format_version(found, template), text.count("\n", 0, found.start()) + 1
+
+
+def describe_form(tag: str, distance: int, pin: str) -> str:
+    """core_version of a tag basis: the tag, or git describe's long form."""
+    return tag if distance == 0 else f"{tag}-{distance}-g{pin[:7]}"
+
+
+def _repo_and_pin(
+    profile: dict, basis: dict, mode: str, cache_dir: str, offline: bool,
+) -> tuple[upstream.Repo, str]:
+    """The repository a basis reads and the commit it is read at."""
+    if basis.get("url"):
+        repo = upstream.parse_repo(str(basis["url"]))
+        pin = str(basis.get("commit") or "")
+        where = str(basis["url"])
+    else:
+        field = str(basis.get("repo") or "source")
+        url = _by_mode(profile, field, mode) or ""
+        repo = upstream.parse_repo(url)
+        pin = _by_mode(profile, PIN_FIELDS.get(field, ""), mode) or ""
+        where = url or field
+        if repo is not None and not pin and field == "upstream" and profile.get("profiled_date"):
+            # The pin of a second repository falls back on profiled_date,
+            # as profile_sync resolves it.
+            pin = upstream.resolve_commit_at(
+                repo, str(profile["profiled_date"]), cache_dir, offline) or ""
+    if repo is None:
+        raise VersionUnreadable(f"{where}: not a forge repository")
+    if not pin:
+        raise VersionUnreadable(f"{repo.slug}: no pinned commit")
+    return repo, pin
+
+
+def _ancestry(
+    repo: upstream.Repo, base: str, head: str, cache_dir: str, offline: bool,
+) -> tuple[int, int] | None:
+    """The forge's compare, else the commit graph: GitHub refuses with 422
+    a compare whose diff is too large (aps3e's tag and pin)."""
+    try:
+        relation = upstream.ancestry(repo, base, head, cache_dir, offline)
+    except upstream.RateLimitError:
+        raise
+    except upstream.UpstreamError:
+        relation = None
+    if relation is None:
+        try:
+            relation = upstream.graph_ancestry(repo, base, head, cache_dir, offline)
+        except upstream.NotCachedError:
+            return None
+    return relation
+
+
+def read(
+    profile: dict, mode: str, basis: dict, cache_dir: str, offline: bool,
+    profiles: dict | None = None, recompute: bool = False,
+) -> Reading:
+    """The version the basis yields at the profile's pin, against core_version.
+
+    ``recompute`` asks git for the nearest tag again, the way a pin advance
+    has to: a recorded tag only proves its own distance, and a tag released
+    between the two pins would go unseen.
+    """
+    stated = declared(profile, mode)
+    kind = str(basis.get("kind") or "")
+
+    def reading(value: str | None, where: str, updates: tuple = ()) -> Reading:
+        status = MATCH if value == stated else DRIFT
+        return Reading(mode, kind, stated, value, where, status, updates)
+
+    def unreadable(reason: str) -> Reading:
+        return Reading(mode, kind, stated, None, reason, UNREADABLE)
+
+    try:
+        if kind == "none":
+            if recompute:
+                # A release tagged since the basis was written gives the
+                # code a version: the advance has to name it.
+                repo, pin = _repo_and_pin(profile, basis, mode, cache_dir, offline)
+                described = upstream.describe(repo, pin, cache_dir, offline)
+                if described is not None:
+                    return Reading(mode, kind, stated, describe_form(*described, pin),
+                                   f"{repo.slug}: tag {described[0]} is now in "
+                                   f"{pin[:7]}'s history", DRIFT)
+            return reading(None, "the code states no version")
+        if kind == "unverified":
+            return Reading(mode, kind, stated, None, f"not verified: {basis.get('why')}",
+                           UNCHECKED)
+        if kind == "package":
+            artifact = str(basis.get("artifact") or "")
+            if basis.get("pattern"):
+                # An artefact named after its version says it in its name:
+                # advancemame-0.94.0.tar.gz.
+                found = read_text_version(artifact, str(basis["pattern"]), basis.get("format"))
+                if found is None:
+                    return unreadable(f"{artifact}: pattern not found")
+                return reading(found[0], f"named by {artifact}")
+            return Reading(mode, kind, stated, None, f"read from {artifact}", UNCHECKED)
+        if kind == "alias":
+            other = (profiles or {}).get(str(basis.get("of") or ""))
+            if other is None:
+                return unreadable(f"alias of {basis.get('of')}: no such profile")
+            return reading(declared(other, mode) if isinstance(other.get("core_version"), dict)
+                           else declared(other, ""), f"alias of {basis['of']}")
+        repo, pin = _repo_and_pin(profile, basis, mode, cache_dir, offline)
+        if kind == "file":
+            path = str(basis.get("path") or "")
+            lines = upstream.fetch_file(repo, pin, path, cache_dir, offline)
+            if lines is None:
+                return unreadable(f"{repo.slug}@{pin[:7]}:{path}: absent")
+            found = read_text_version("\n".join(lines), str(basis.get("pattern") or ""),
+                                      basis.get("format"))
+            if found is None:
+                return unreadable(f"{repo.slug}@{pin[:7]}:{path}: pattern not found")
+            value, line = found
+            return reading(value, f"{repo.slug}@{pin[:7]}:{path}:{line}")
+        if kind == "tag":
+            if recompute:
+                described = upstream.describe(repo, pin, cache_dir, offline, basis.get("match"))
+                if described is None:
+                    return unreadable(f"{repo.slug}@{pin[:7]}: no tag in its history")
+                tag, distance = described
+            else:
+                tag, distance = str(basis.get("tag") or ""), basis.get("distance")
+                commit = upstream.tag_commit(repo, tag, cache_dir, offline)
+                if commit is None:
+                    return unreadable(f"{repo.slug}: no tag {tag}")
+                relation = (0, 0) if commit == pin else _ancestry(
+                    repo, commit, pin, cache_dir, offline)
+                if relation is None:
+                    return unreadable(f"{repo.slug}: {tag}...{pin[:7]} not comparable")
+                if relation[1]:
+                    return unreadable(f"{repo.slug}: tag {tag} is not in {pin[:7]}'s history")
+                if relation[0] != distance:
+                    return Reading(mode, kind, stated, describe_form(tag, relation[0], pin),
+                                   f"{repo.slug}: {pin[:7]} is {relation[0]} commits after "
+                                   f"{tag}, the basis says {distance}", DRIFT,
+                                   (("distance", relation[0]),))
+            return reading(describe_form(tag, distance, pin),
+                           f"{repo.slug}: tag {tag} + {distance}",
+                           (("tag", tag), ("distance", distance)))
+    except (upstream.UpstreamError, VersionUnreadable) as exc:
+        return unreadable(str(exc))
+    return unreadable(f"unknown kind {kind!r}")
+
+
+def check(
+    profile: dict, cache_dir: str, offline: bool, profiles: dict | None = None,
+    recompute: bool = False,
+) -> list[Reading]:
+    """One reading per build mode, or a NO_BASIS reading."""
+    found = bases(profile)
+    if not found:
+        return [Reading("", "", declared(profile, ""), None, "no version_basis", NO_BASIS)]
+    return [read(profile, mode, basis, cache_dir, offline, profiles, recompute)
+            for mode, basis in found.items()]