mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 00:27:20 -05:00
feat: read core_version from its recorded basis
version_basis says where a profile's version is read: a pattern in a file, the nearest tag and its distance, an alias, a package, none, or unverified with the reason. profile_sync --verify-version reads it again at the pin and exits 1 on a difference, --set-version writes what the code states. A pin advance reads the version at the new pin in the same write and is refused when it cannot. upstream.describe counts the commits after the nearest tag in a treeless clone kept under the cache, and graph_ancestry answers a pair GitHub refuses to compare.
This commit is contained in:
1 parent
1f6c018006
commit
6aa236440a
4 files changed
+649
-3
No files matched your search
+252
-3
@@ -31,6 +31,7 @@ import yaml
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
import upstream
|
||||
import version_basis
|
||||
from common import (
|
||||
_norm_system_id,
|
||||
github_token,
|
||||
@@ -2670,6 +2671,128 @@ def apply_edit(path: Path, new_text: str, expected: dict) -> None:
|
||||
raise YamlWriteError(f"{path}: structure changed beyond the intended field")
|
||||
|
||||
|
||||
# Key order of a written version_basis, as a reader expects it.
|
||||
_BASIS_ORDER = ("kind", "why", "repo", "url", "commit", "path", "pattern", "format",
|
||||
"match", "tag", "distance", "of", "artifact")
|
||||
|
||||
|
||||
def _top_block(lines: list[str], key: str) -> tuple[int, int] | None:
|
||||
"""Line span of a top-level key and the indented lines under it."""
|
||||
indices = _top_level_indices(lines, key)
|
||||
if not indices:
|
||||
return None
|
||||
if len(indices) > 1:
|
||||
raise YamlWriteError(f"{key}: expected one top-level line, found {len(indices)}")
|
||||
end = indices[0] + 1
|
||||
while end < len(lines) and (lines[end].startswith((" ", "\t")) or not lines[end].strip()):
|
||||
end += 1
|
||||
# Blank lines closing the block belong to the document, not to it.
|
||||
while end > indices[0] + 1 and not lines[end - 1].strip():
|
||||
end -= 1
|
||||
return indices[0], end
|
||||
|
||||
|
||||
def _render_version(value: object) -> list[str]:
|
||||
if isinstance(value, dict):
|
||||
return ["core_version:"] + [
|
||||
f" {mode}: {'null' if v is None else _double_quoted(str(v))}"
|
||||
for mode, v in value.items()
|
||||
]
|
||||
return [f"core_version: {'null' if value is None else _double_quoted(str(value))}"]
|
||||
|
||||
|
||||
def _render_basis(basis: dict) -> list[str]:
|
||||
def ordered(entry: dict) -> dict:
|
||||
rank = {key: i for i, key in enumerate(_BASIS_ORDER)}
|
||||
return dict(sorted(entry.items(), key=lambda item: rank.get(item[0], len(rank))))
|
||||
|
||||
def scalar(value: object) -> str:
|
||||
# Every string quoted: a tag 2407 or 0.10 read back as a number. A
|
||||
# pattern keeps its backslashes readable in single quotes.
|
||||
if isinstance(value, bool) or not isinstance(value, str):
|
||||
return str(value).lower() if isinstance(value, bool) else str(value)
|
||||
if "\\" in value or '"' in value:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
return _double_quoted(value)
|
||||
|
||||
def block(entry: dict, indent: str) -> list[str]:
|
||||
# kind, why and repo are words of the schema's enums, written bare.
|
||||
return [f"{indent}{key}: {value if key in ('kind', 'why', 'repo') else scalar(value)}"
|
||||
for key, value in ordered(entry).items()]
|
||||
|
||||
if "kind" in basis:
|
||||
return ["version_basis:", *block(basis, " ")]
|
||||
lines = ["version_basis:"]
|
||||
for mode, entry in basis.items():
|
||||
lines += [f" {mode}:", *block(entry, " ")]
|
||||
return lines
|
||||
|
||||
|
||||
def set_version_fields(text: str, version: object, basis: dict | None) -> str:
|
||||
"""Rewrite core_version and version_basis, version_basis right after it."""
|
||||
lines = text.splitlines()
|
||||
span = _top_block(lines, "core_version")
|
||||
rendered = _render_version(version)
|
||||
if span is None:
|
||||
anchor = _top_block(lines, "upstream_commit") or _top_block(lines, "source_commit") \
|
||||
or _top_block(lines, "profiled_date") or _top_block(lines, "upstream") \
|
||||
or _top_block(lines, "source")
|
||||
if anchor is None:
|
||||
raise YamlWriteError("core_version: no line to write it after")
|
||||
lines[anchor[1]:anchor[1]] = rendered
|
||||
else:
|
||||
lines[span[0]:span[1]] = rendered
|
||||
if basis is not None:
|
||||
old = _top_block(lines, "version_basis")
|
||||
if old is not None:
|
||||
del lines[old[0]:old[1]]
|
||||
after = _top_block(lines, "core_version")
|
||||
lines[after[1]:after[1]] = _render_basis(basis)
|
||||
return "\n".join(lines) + ("\n" if text.endswith("\n") else "")
|
||||
|
||||
|
||||
def apply_readings(document: dict, readings: list) -> tuple[object, dict | None]:
|
||||
"""core_version and version_basis as the readings state them.
|
||||
|
||||
A reading that could not be made leaves its build mode as written.
|
||||
"""
|
||||
version = document.get("core_version")
|
||||
basis = document.get("version_basis")
|
||||
basis = dict(basis) if isinstance(basis, dict) else None
|
||||
if isinstance(version, dict):
|
||||
version = dict(version)
|
||||
for reading in readings:
|
||||
if reading.status not in (version_basis.MATCH, version_basis.DRIFT):
|
||||
continue
|
||||
if reading.mode:
|
||||
if not isinstance(version, dict):
|
||||
version = {}
|
||||
version[reading.mode] = reading.read
|
||||
else:
|
||||
version = reading.read
|
||||
if reading.updates and basis is not None:
|
||||
target = basis if not reading.mode else dict(basis.get(reading.mode) or {})
|
||||
target.update(dict(reading.updates))
|
||||
if reading.mode:
|
||||
basis[reading.mode] = target
|
||||
return version, basis
|
||||
|
||||
|
||||
def write_version(path: Path, readings: list) -> bool:
|
||||
"""Set core_version (and a tag basis' tag and distance) from the readings."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
document = yaml.safe_load(text)
|
||||
version, basis = apply_readings(document, readings)
|
||||
if version == document.get("core_version") and basis == document.get("version_basis"):
|
||||
return False
|
||||
expected = dict(document)
|
||||
expected["core_version"] = version
|
||||
if basis is not None:
|
||||
expected["version_basis"] = basis
|
||||
apply_edit(path, set_version_fields(text, version, basis), expected)
|
||||
return True
|
||||
|
||||
|
||||
def backfill_commit(path: Path, sha: str) -> bool:
|
||||
"""Insert source_commit after profiled_date. False when already present."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
@@ -3313,9 +3436,15 @@ def _needs_review(report: ProfileReport, accept_changed: bool) -> bool:
|
||||
|
||||
|
||||
def bump_commit(
|
||||
path: Path, report: ProfileReport, accept_changed: bool = False
|
||||
path: Path, report: ProfileReport, accept_changed: bool = False,
|
||||
cache_dir: str = DEFAULT_CACHE, offline: bool = False,
|
||||
) -> bool:
|
||||
"""Advance source_commit to HEAD when nothing needs a read again."""
|
||||
"""Advance source_commit to HEAD when nothing needs a read again.
|
||||
|
||||
core_version moves with the pin, read again from version_basis at the
|
||||
new revision. Four bulk advances moved 295 pins and no version: an
|
||||
advance whose version cannot be read is refused.
|
||||
"""
|
||||
if _needs_review(report, accept_changed):
|
||||
return False
|
||||
text = path.read_text(encoding="utf-8")
|
||||
@@ -3375,10 +3504,33 @@ def bump_commit(
|
||||
new_text = _set_pin(
|
||||
new_text, document, "upstream_commit", report.upstream_head, "source_commit"
|
||||
)
|
||||
readings = version_basis.check(expected, cache_dir, offline, recompute=True)
|
||||
refused = version_refusal(readings)
|
||||
if refused:
|
||||
print(f"{path.stem}: {refused}, its pin is not advanced", file=sys.stderr)
|
||||
return False
|
||||
version, basis = apply_readings(expected, readings)
|
||||
if version != expected.get("core_version") or basis != expected.get("version_basis"):
|
||||
new_text = set_version_fields(new_text, version, basis)
|
||||
expected["core_version"] = version
|
||||
if basis is not None:
|
||||
expected["version_basis"] = basis
|
||||
apply_edit(path, new_text, expected)
|
||||
return True
|
||||
|
||||
|
||||
def version_refusal(readings: list) -> str | None:
|
||||
"""Why the version at a new pin cannot be stated, or None."""
|
||||
for reading in readings:
|
||||
if reading.status == version_basis.NO_BASIS:
|
||||
return "no version_basis says where its version is read"
|
||||
if reading.status == version_basis.UNREADABLE:
|
||||
return f"core_version cannot be read at the new pin: {reading.where}"
|
||||
if reading.status == version_basis.UNCHECKED:
|
||||
return f"core_version is {reading.where}, not read from the code at a pin"
|
||||
return None
|
||||
|
||||
|
||||
def pins_changed(before: dict, after: dict) -> list[str]:
|
||||
"""The pin fields whose value a write changed."""
|
||||
return [
|
||||
@@ -3996,6 +4148,16 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
"--check-version", action="store_true",
|
||||
help="compare core_version with the latest tag and release",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verify-version", action="store_true",
|
||||
help="read core_version again from version_basis at the pin and "
|
||||
"exit 1 when the code states another",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--set-version", action="store_true",
|
||||
help="write core_version from version_basis at the pin, the nearest "
|
||||
"tag asked of git again",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detect-new-files", action="store_true",
|
||||
help="name files upstream loads that the profile does not list",
|
||||
@@ -4284,7 +4446,8 @@ def _apply_writes(
|
||||
applied = []
|
||||
if args.rebase_refs and not report.skipped:
|
||||
applied = rebase_refs(target, report, args.accept_changed)
|
||||
moved = bump_commit(target, report, args.accept_changed)
|
||||
moved = bump_commit(target, report, args.accept_changed,
|
||||
args.cache_dir, args.offline)
|
||||
if args.rebase_refs and applied and not moved:
|
||||
keyed = any(isinstance(profile.get(f), dict) for f in ("source_commit", "upstream_commit"))
|
||||
reason = (
|
||||
@@ -4723,6 +4886,32 @@ def _print_triage(
|
||||
|
||||
def _refuse_flag_combinations(args: argparse.Namespace) -> None:
|
||||
"""Exit on a flag the chosen mode would not apply, before any request."""
|
||||
if args.verify_version or args.set_version:
|
||||
ignored = [
|
||||
flag for flag, on in (
|
||||
("--verify-version", args.verify_version and args.set_version),
|
||||
("--markdown", args.markdown),
|
||||
("--fetch-plan", args.fetch_plan),
|
||||
("--triage", args.triage),
|
||||
("--strict", args.strict),
|
||||
("--check-version", args.check_version),
|
||||
("--detect-new-files", args.detect_new_files),
|
||||
("--watch-hashes", args.watch_hashes),
|
||||
("--full-diff", args.full_diff),
|
||||
("--tree-diff", args.tree_diff),
|
||||
("--backfill-commits", args.backfill_commits),
|
||||
("--rebase-refs", args.rebase_refs),
|
||||
("--attribute-refs", args.attribute_refs),
|
||||
("--bump-commit", args.bump_commit),
|
||||
("--realign-prose", args.realign_prose),
|
||||
("--accept-changed", args.accept_changed),
|
||||
("--dry-run", args.dry_run and not args.set_version),
|
||||
) if on
|
||||
]
|
||||
if ignored:
|
||||
mode = "--set-version" if args.set_version else "--verify-version"
|
||||
print(f"{mode} does not apply {', '.join(ignored)}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
if args.fresh and args.offline:
|
||||
print("--fresh asks the forges again, which --offline forbids.", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -4809,6 +4998,63 @@ def _refuse_flag_combinations(args: argparse.Namespace) -> None:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _version_selection(args: argparse.Namespace, profiles: dict) -> dict:
|
||||
"""--all reaches the alias and test profiles too: each states a version."""
|
||||
if args.all and not (args.system or args.type_ or args.stale_before or args.limit):
|
||||
return dict(sorted(profiles.items()))
|
||||
return select_profiles(profiles, args)
|
||||
|
||||
|
||||
def format_reading(name: str, reading: version_basis.Reading) -> str:
|
||||
mode = f"[{reading.mode}] " if reading.mode else ""
|
||||
kind = f"{reading.kind} " if reading.kind else ""
|
||||
if reading.status == version_basis.DRIFT:
|
||||
said = f"declared {reading.declared}, code {reading.read}"
|
||||
elif reading.status == version_basis.MATCH:
|
||||
said = str(reading.read)
|
||||
else:
|
||||
said = f"declared {reading.declared}"
|
||||
return f"{name}: {mode}{reading.status} {said} ({kind}{reading.where})"
|
||||
|
||||
|
||||
def run_versions(args: argparse.Namespace, profiles: dict) -> None:
|
||||
"""--verify-version and --set-version: core_version against the code."""
|
||||
selected = _version_selection(args, profiles)
|
||||
if args.set_version and not args.dry_run and not args.force:
|
||||
_refuse_dirty_tree(args.emulators_dir, list(selected))
|
||||
counts: Counter = Counter()
|
||||
rows = []
|
||||
failed = False
|
||||
for name, profile in selected.items():
|
||||
readings = version_basis.check(
|
||||
profile, args.cache_dir, args.offline, profiles, recompute=args.set_version
|
||||
)
|
||||
for reading in readings:
|
||||
counts[reading.status] += 1
|
||||
rows.append({"profile": name, **asdict(reading)})
|
||||
if not args.as_json and (not args.changed_only or reading.status != version_basis.MATCH):
|
||||
print(format_reading(name, reading))
|
||||
if args.set_version:
|
||||
path = Path(args.emulators_dir) / f"{name}.yml"
|
||||
if args.dry_run:
|
||||
continue
|
||||
try:
|
||||
if write_version(path, readings):
|
||||
print(f"{name}: core_version written")
|
||||
except YamlWriteError as exc:
|
||||
print(f"{name}: write refused: {exc}", file=sys.stderr)
|
||||
failed = True
|
||||
elif any(r.status in version_basis.FAILING for r in readings):
|
||||
failed = True
|
||||
if args.as_json:
|
||||
print(json.dumps(rows, indent=2))
|
||||
else:
|
||||
print(", ".join(f"{count} {status}" for status, count in sorted(counts.items())),
|
||||
file=sys.stderr)
|
||||
if failed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Build one report per selected profile, then apply writes and output."""
|
||||
args = build_parser().parse_args()
|
||||
@@ -4818,6 +5064,9 @@ def main() -> None:
|
||||
# trzy/Supermodel and VICE read as behind their real HEAD.
|
||||
upstream.MOVING_TTL = 0
|
||||
profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
|
||||
if args.verify_version or args.set_version:
|
||||
run_versions(args, profiles)
|
||||
return
|
||||
selected = select_profiles(profiles, args)
|
||||
_check_quota(len(selected), args.offline)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import http.client
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -1086,3 +1087,103 @@ def commits_touching(
|
||||
1 for entry in payload
|
||||
if isinstance(entry, dict) and (entry.get("sha") or entry.get("id")) != base
|
||||
)
|
||||
|
||||
|
||||
# Commit graphs kept to count the commits between a tag and a pin. A
|
||||
# treeless bare clone holds commits and tags only: the files a profile reads
|
||||
# come from fetch_file, by sha.
|
||||
GIT_DIR = "_git"
|
||||
|
||||
|
||||
def clone_dir(cache_dir: str, repo: Repo) -> Path:
|
||||
"""Where the commit graph of a repository is kept."""
|
||||
return Path(cache_dir) / GIT_DIR / repo.host / repo.owner / f"{repo.name}.git"
|
||||
|
||||
|
||||
def _git(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess:
|
||||
env = dict(os.environ, GIT_TERMINAL_PROMPT="0")
|
||||
return subprocess.run(
|
||||
["git", *args], cwd=cwd, capture_output=True, text=True, env=env, check=False
|
||||
)
|
||||
|
||||
|
||||
def _has_commit(clone: Path, sha: str) -> bool:
|
||||
return _git(["cat-file", "-e", f"{sha}^{{commit}}"], clone).returncode == 0
|
||||
|
||||
|
||||
def _graph(repo: Repo, sha: str, cache_dir: str, offline: bool) -> Path:
|
||||
"""A clone holding sha and every tag, fetched again once tags may have moved."""
|
||||
clone = clone_dir(cache_dir, repo)
|
||||
url = f"https://{repo.host}/{repo.slug}.git"
|
||||
if not (clone / "HEAD").is_file():
|
||||
if offline:
|
||||
raise NotCachedError(f"{repo.slug}: no commit graph in the offline cache")
|
||||
clone.parent.mkdir(parents=True, exist_ok=True)
|
||||
result = _git(["clone", "--quiet", "--bare", "--filter=tree:0", url, str(clone)])
|
||||
if result.returncode:
|
||||
raise UpstreamError(f"{url}: clone failed: {result.stderr.strip()}")
|
||||
_git(["fetch", "--quiet", "--filter=tree:0", "--tags", "origin"], clone)
|
||||
stamp = clone / "FETCH_HEAD"
|
||||
fresh = stamp.is_file() and time.time() - stamp.stat().st_mtime <= MOVING_TTL
|
||||
if offline or (fresh and _has_commit(clone, sha)):
|
||||
if not _has_commit(clone, sha):
|
||||
raise NotCachedError(f"{repo.slug}@{sha[:8]}: not in the offline commit graph")
|
||||
return clone
|
||||
# Tags are fetched again: a tag pushed since the clone can be nearer.
|
||||
result = _git(
|
||||
["fetch", "--quiet", "--filter=tree:0", "--force", "--tags", "origin",
|
||||
"+refs/heads/*:refs/heads/*"], clone,
|
||||
)
|
||||
if result.returncode:
|
||||
raise UpstreamError(f"{url}: fetch failed: {result.stderr.strip()}")
|
||||
if not _has_commit(clone, sha):
|
||||
# A commit no branch reaches any more: a rewritten fork, a PR head.
|
||||
_git(["fetch", "--quiet", "--filter=tree:0", "origin", sha], clone)
|
||||
if not _has_commit(clone, sha):
|
||||
raise UpstreamError(f"{url}: {sha[:8]} is on no branch the forge serves")
|
||||
return clone
|
||||
|
||||
|
||||
def describe(
|
||||
repo: Repo, sha: str, cache_dir: str, offline: bool = False,
|
||||
match: str | None = None,
|
||||
) -> tuple[str, int] | None:
|
||||
"""The tag nearest to sha in its history, and the commits after it.
|
||||
|
||||
git describe counts the commits sha has that the tag lacks, the count a
|
||||
forge compare reports as ahead. ``match`` is a glob the tag must fit.
|
||||
None when no tag is reachable from sha.
|
||||
"""
|
||||
clone = _graph(repo, sha, cache_dir, offline)
|
||||
argv = ["describe", "--tags", "--long", "--abbrev=40"]
|
||||
if match:
|
||||
argv += ["--match", match]
|
||||
result = _git([*argv, sha], clone)
|
||||
if result.returncode:
|
||||
if any(text in result.stderr for text in (
|
||||
"No names found", "No tags can describe", "cannot describe",
|
||||
)):
|
||||
return None
|
||||
raise UpstreamError(f"{repo.slug}: describe {sha[:8]}: {result.stderr.strip()}")
|
||||
tag, _distance, _commit = result.stdout.strip().rsplit("-", 2)
|
||||
# describe's own count is an estimate across merges (ares: 149 where the
|
||||
# pin holds 143 commits the tag lacks): the exact count is rev-list's.
|
||||
counted = _git(["rev-list", "--count", f"refs/tags/{tag}..{sha}"], clone)
|
||||
if counted.returncode:
|
||||
raise UpstreamError(f"{repo.slug}: {tag}..{sha[:8]}: {counted.stderr.strip()}")
|
||||
return tag, int(counted.stdout.strip())
|
||||
|
||||
|
||||
def graph_ancestry(
|
||||
repo: Repo, base: str, head: str, cache_dir: str, offline: bool = False
|
||||
) -> tuple[int, int]:
|
||||
"""(ahead, behind) counted in the commit graph, for a pair a forge
|
||||
refuses to compare (GitHub answers 422 past a diff size)."""
|
||||
clone = _graph(repo, head, cache_dir, offline)
|
||||
counts = []
|
||||
for spec in (f"{base}..{head}", f"{head}..{base}"):
|
||||
result = _git(["rev-list", "--count", spec], clone)
|
||||
if result.returncode:
|
||||
raise UpstreamError(f"{repo.slug}: {spec}: {result.stderr.strip()}")
|
||||
counts.append(int(result.stdout.strip()))
|
||||
return counts[0], counts[1]
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Read a profile's core_version from the source its version_basis records.
|
||||
|
||||
A profile states the version of the code it was read at. Copied from a
|
||||
libretro .info or left behind by a pin advance, it named another release:
|
||||
snes9x said 1.61 at a pin whose snes9x.h states 1.63, and four bulk advances
|
||||
moved 295 pins without touching one version. version_basis records where the
|
||||
version is read, so a tool can read it again at any pin:
|
||||
|
||||
- ``file``: a pattern in one file of a declared repository at its pin, the
|
||||
version the program states (a macro, a build file, a package file).
|
||||
- ``tag``: the nearest tag in the pin's history and the commits after it,
|
||||
for code that takes its version from git at build time. core_version is
|
||||
the tag at distance 0, else ``<tag>-<distance>-g<pin7>``, git describe's
|
||||
form.
|
||||
- ``alias``: the same binary as another profile, whose version it carries.
|
||||
- ``package``: an artefact read outside any source tree (an Android package,
|
||||
a closed binary, a release archive): a version no pin can be asked for.
|
||||
A pattern reads it from the artefact's name when the name states it.
|
||||
- ``none``: the code at the pin states no version and no tag reaches it.
|
||||
core_version is then null.
|
||||
- ``unverified``: no reading could be made (``why``: no pin, a host the
|
||||
tools cannot read, a forge gone, no build named as read, no source).
|
||||
core_version stays as written and is reported unconfirmed.
|
||||
|
||||
A profile whose builds live in different repositories keys version_basis by
|
||||
build mode, the way it keys core_version.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import upstream
|
||||
|
||||
KINDS = ("file", "tag", "alias", "package", "none", "unverified")
|
||||
# Values copied from a .info or a branch name, which name no version.
|
||||
PLACEHOLDERS = frozenset({"git", "svn", "unknown", "master", "main", "latest", "dev", "head"})
|
||||
# Pin field of each repository field a basis may name. A mirror carries the
|
||||
# source's history, so source_commit pins it too.
|
||||
PIN_FIELDS = {"source": "source_commit", "source_mirror": "source_commit",
|
||||
"upstream": "upstream_commit"}
|
||||
|
||||
MATCH = "MATCH"
|
||||
DRIFT = "DRIFT"
|
||||
UNREADABLE = "UNREADABLE"
|
||||
NO_BASIS = "NO_BASIS"
|
||||
UNCHECKED = "UNCHECKED"
|
||||
FAILING = (DRIFT, UNREADABLE, NO_BASIS)
|
||||
|
||||
_FIELD = re.compile(r"\{(\d+)(?::([^}]*))?\}")
|
||||
|
||||
|
||||
class VersionUnreadable(Exception):
|
||||
"""The basis yields no version at the pin it is asked about."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Reading:
|
||||
mode: str
|
||||
kind: str
|
||||
declared: str | None
|
||||
read: str | None
|
||||
where: str
|
||||
status: str
|
||||
# What a write sets beside core_version: the tag and distance of a tag basis.
|
||||
updates: tuple[tuple[str, object], ...] = ()
|
||||
|
||||
|
||||
def bases(profile: dict) -> dict[str, dict]:
|
||||
"""The basis of each build mode, "" when one covers every build."""
|
||||
basis = profile.get("version_basis")
|
||||
if not isinstance(basis, dict):
|
||||
return {}
|
||||
if "kind" in basis:
|
||||
return {"": basis}
|
||||
return {str(mode): value for mode, value in basis.items() if isinstance(value, dict)}
|
||||
|
||||
|
||||
def declared(profile: dict, mode: str) -> str | None:
|
||||
"""core_version for a build mode, as written."""
|
||||
value = profile.get("core_version")
|
||||
if isinstance(value, dict):
|
||||
value = value.get(mode)
|
||||
return None if value is None else str(value)
|
||||
|
||||
|
||||
def _by_mode(profile: dict, field: str, mode: str) -> str | None:
|
||||
value = profile.get(field)
|
||||
if isinstance(value, dict):
|
||||
value = value.get(mode)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def is_placeholder(value: object) -> bool:
|
||||
return isinstance(value, str) and value.strip().casefold() in PLACEHOLDERS
|
||||
|
||||
|
||||
def format_version(match: re.Match, template: str | None) -> str:
|
||||
"""The groups of a match laid out by template, `{1}.{2}` or `{4:02d}`.
|
||||
|
||||
A format spec applies to the group read as an integer: FBNeo writes its
|
||||
alpha number on two digits (VER_ALPHA 3 is 1.0.0.03).
|
||||
"""
|
||||
if not template:
|
||||
return match.group(1).strip()
|
||||
|
||||
def field(found: re.Match) -> str:
|
||||
text = match.group(int(found.group(1))) or ""
|
||||
spec = found.group(2)
|
||||
return format(int(text, 0), spec) if spec else text.strip()
|
||||
|
||||
return _FIELD.sub(field, template)
|
||||
|
||||
|
||||
def read_text_version(text: str, pattern: str, template: str | None) -> tuple[str, int] | None:
|
||||
"""The version a pattern reads in a file's text, and its line number."""
|
||||
found = re.search(pattern, text, re.MULTILINE)
|
||||
if not found:
|
||||
return None
|
||||
return format_version(found, template), text.count("\n", 0, found.start()) + 1
|
||||
|
||||
|
||||
def describe_form(tag: str, distance: int, pin: str) -> str:
|
||||
"""core_version of a tag basis: the tag, or git describe's long form."""
|
||||
return tag if distance == 0 else f"{tag}-{distance}-g{pin[:7]}"
|
||||
|
||||
|
||||
def _repo_and_pin(
|
||||
profile: dict, basis: dict, mode: str, cache_dir: str, offline: bool,
|
||||
) -> tuple[upstream.Repo, str]:
|
||||
"""The repository a basis reads and the commit it is read at."""
|
||||
if basis.get("url"):
|
||||
repo = upstream.parse_repo(str(basis["url"]))
|
||||
pin = str(basis.get("commit") or "")
|
||||
where = str(basis["url"])
|
||||
else:
|
||||
field = str(basis.get("repo") or "source")
|
||||
url = _by_mode(profile, field, mode) or ""
|
||||
repo = upstream.parse_repo(url)
|
||||
pin = _by_mode(profile, PIN_FIELDS.get(field, ""), mode) or ""
|
||||
where = url or field
|
||||
if repo is not None and not pin and field == "upstream" and profile.get("profiled_date"):
|
||||
# The pin of a second repository falls back on profiled_date,
|
||||
# as profile_sync resolves it.
|
||||
pin = upstream.resolve_commit_at(
|
||||
repo, str(profile["profiled_date"]), cache_dir, offline) or ""
|
||||
if repo is None:
|
||||
raise VersionUnreadable(f"{where}: not a forge repository")
|
||||
if not pin:
|
||||
raise VersionUnreadable(f"{repo.slug}: no pinned commit")
|
||||
return repo, pin
|
||||
|
||||
|
||||
def _ancestry(
|
||||
repo: upstream.Repo, base: str, head: str, cache_dir: str, offline: bool,
|
||||
) -> tuple[int, int] | None:
|
||||
"""The forge's compare, else the commit graph: GitHub refuses with 422
|
||||
a compare whose diff is too large (aps3e's tag and pin)."""
|
||||
try:
|
||||
relation = upstream.ancestry(repo, base, head, cache_dir, offline)
|
||||
except upstream.RateLimitError:
|
||||
raise
|
||||
except upstream.UpstreamError:
|
||||
relation = None
|
||||
if relation is None:
|
||||
try:
|
||||
relation = upstream.graph_ancestry(repo, base, head, cache_dir, offline)
|
||||
except upstream.NotCachedError:
|
||||
return None
|
||||
return relation
|
||||
|
||||
|
||||
def read(
|
||||
profile: dict, mode: str, basis: dict, cache_dir: str, offline: bool,
|
||||
profiles: dict | None = None, recompute: bool = False,
|
||||
) -> Reading:
|
||||
"""The version the basis yields at the profile's pin, against core_version.
|
||||
|
||||
``recompute`` asks git for the nearest tag again, the way a pin advance
|
||||
has to: a recorded tag only proves its own distance, and a tag released
|
||||
between the two pins would go unseen.
|
||||
"""
|
||||
stated = declared(profile, mode)
|
||||
kind = str(basis.get("kind") or "")
|
||||
|
||||
def reading(value: str | None, where: str, updates: tuple = ()) -> Reading:
|
||||
status = MATCH if value == stated else DRIFT
|
||||
return Reading(mode, kind, stated, value, where, status, updates)
|
||||
|
||||
def unreadable(reason: str) -> Reading:
|
||||
return Reading(mode, kind, stated, None, reason, UNREADABLE)
|
||||
|
||||
try:
|
||||
if kind == "none":
|
||||
if recompute:
|
||||
# A release tagged since the basis was written gives the
|
||||
# code a version: the advance has to name it.
|
||||
repo, pin = _repo_and_pin(profile, basis, mode, cache_dir, offline)
|
||||
described = upstream.describe(repo, pin, cache_dir, offline)
|
||||
if described is not None:
|
||||
return Reading(mode, kind, stated, describe_form(*described, pin),
|
||||
f"{repo.slug}: tag {described[0]} is now in "
|
||||
f"{pin[:7]}'s history", DRIFT)
|
||||
return reading(None, "the code states no version")
|
||||
if kind == "unverified":
|
||||
return Reading(mode, kind, stated, None, f"not verified: {basis.get('why')}",
|
||||
UNCHECKED)
|
||||
if kind == "package":
|
||||
artifact = str(basis.get("artifact") or "")
|
||||
if basis.get("pattern"):
|
||||
# An artefact named after its version says it in its name:
|
||||
# advancemame-0.94.0.tar.gz.
|
||||
found = read_text_version(artifact, str(basis["pattern"]), basis.get("format"))
|
||||
if found is None:
|
||||
return unreadable(f"{artifact}: pattern not found")
|
||||
return reading(found[0], f"named by {artifact}")
|
||||
return Reading(mode, kind, stated, None, f"read from {artifact}", UNCHECKED)
|
||||
if kind == "alias":
|
||||
other = (profiles or {}).get(str(basis.get("of") or ""))
|
||||
if other is None:
|
||||
return unreadable(f"alias of {basis.get('of')}: no such profile")
|
||||
return reading(declared(other, mode) if isinstance(other.get("core_version"), dict)
|
||||
else declared(other, ""), f"alias of {basis['of']}")
|
||||
repo, pin = _repo_and_pin(profile, basis, mode, cache_dir, offline)
|
||||
if kind == "file":
|
||||
path = str(basis.get("path") or "")
|
||||
lines = upstream.fetch_file(repo, pin, path, cache_dir, offline)
|
||||
if lines is None:
|
||||
return unreadable(f"{repo.slug}@{pin[:7]}:{path}: absent")
|
||||
found = read_text_version("\n".join(lines), str(basis.get("pattern") or ""),
|
||||
basis.get("format"))
|
||||
if found is None:
|
||||
return unreadable(f"{repo.slug}@{pin[:7]}:{path}: pattern not found")
|
||||
value, line = found
|
||||
return reading(value, f"{repo.slug}@{pin[:7]}:{path}:{line}")
|
||||
if kind == "tag":
|
||||
if recompute:
|
||||
described = upstream.describe(repo, pin, cache_dir, offline, basis.get("match"))
|
||||
if described is None:
|
||||
return unreadable(f"{repo.slug}@{pin[:7]}: no tag in its history")
|
||||
tag, distance = described
|
||||
else:
|
||||
tag, distance = str(basis.get("tag") or ""), basis.get("distance")
|
||||
commit = upstream.tag_commit(repo, tag, cache_dir, offline)
|
||||
if commit is None:
|
||||
return unreadable(f"{repo.slug}: no tag {tag}")
|
||||
relation = (0, 0) if commit == pin else _ancestry(
|
||||
repo, commit, pin, cache_dir, offline)
|
||||
if relation is None:
|
||||
return unreadable(f"{repo.slug}: {tag}...{pin[:7]} not comparable")
|
||||
if relation[1]:
|
||||
return unreadable(f"{repo.slug}: tag {tag} is not in {pin[:7]}'s history")
|
||||
if relation[0] != distance:
|
||||
return Reading(mode, kind, stated, describe_form(tag, relation[0], pin),
|
||||
f"{repo.slug}: {pin[:7]} is {relation[0]} commits after "
|
||||
f"{tag}, the basis says {distance}", DRIFT,
|
||||
(("distance", relation[0]),))
|
||||
return reading(describe_form(tag, distance, pin),
|
||||
f"{repo.slug}: tag {tag} + {distance}",
|
||||
(("tag", tag), ("distance", distance)))
|
||||
except (upstream.UpstreamError, VersionUnreadable) as exc:
|
||||
return unreadable(str(exc))
|
||||
return unreadable(f"unknown kind {kind!r}")
|
||||
|
||||
|
||||
def check(
|
||||
profile: dict, cache_dir: str, offline: bool, profiles: dict | None = None,
|
||||
recompute: bool = False,
|
||||
) -> list[Reading]:
|
||||
"""One reading per build mode, or a NO_BASIS reading."""
|
||||
found = bases(profile)
|
||||
if not found:
|
||||
return [Reading("", "", declared(profile, ""), None, "no version_basis", NO_BASIS)]
|
||||
return [read(profile, mode, basis, cache_dir, offline, profiles, recompute)
|
||||
for mode, basis in found.items()]
|
||||
@@ -60,9 +60,29 @@ from profile_sync import (
|
||||
worst_status,
|
||||
)
|
||||
import upstream
|
||||
import version_basis
|
||||
from upstream import CompareResult, FileChange
|
||||
|
||||
|
||||
def _version_read_as_written(profile, cache_dir, offline, profiles=None, recompute=False):
|
||||
"""bump_commit reads core_version again at the new pin. These fixtures
|
||||
have no forge behind them, so the version reads as written here:
|
||||
tests/test_version_basis.py holds the reading and the refusals."""
|
||||
stated = version_basis.declared(profile, "")
|
||||
return [version_basis.Reading("", "file", stated, stated, "fixture", version_basis.MATCH)]
|
||||
|
||||
|
||||
_VERSION_AS_WRITTEN = mock.patch.object(version_basis, "check", _version_read_as_written)
|
||||
|
||||
|
||||
def setUpModule():
|
||||
_VERSION_AS_WRITTEN.start()
|
||||
|
||||
|
||||
def tearDownModule():
|
||||
_VERSION_AS_WRITTEN.stop()
|
||||
|
||||
|
||||
class TestParseSourceRef(unittest.TestCase):
|
||||
def test_path_with_single_line(self):
|
||||
self.assertEqual(
|
||||
|
||||
Reference in new issue
Block a user