mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-24 04:39:06 -05:00
492 lines
20 KiB
Python
492 lines
20 KiB
Python
"""Permalinks into the sources a profile cites.
|
|
|
|
A citation is only checkable if it points at the revision the profile
|
|
was written against, so a permalink is pinned, never guessed."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import re
|
|
import urllib.parse
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
import urllib.request
|
|
|
|
import upstream
|
|
from siterender import _by_mode
|
|
from sourceref import _prose_runs, ref_role, source_ref_values, split_source_ref
|
|
|
|
|
|
def _repo_pins(profile: dict, field: str, label: str) -> list[tuple[str, str, str]]:
|
|
"""Pair each declared repository URL with its own revision.
|
|
|
|
A profile whose builds live in separate repositories keys both the URL
|
|
and the revision by build mode. Reading them apart would pin a libretro
|
|
fork to the standalone revision and produce a permalink into a tree that
|
|
never held the cited line. Each tuple is (url, pin, source_commit
|
|
fallback for that same mode).
|
|
"""
|
|
raw = profile.get(field, "")
|
|
pin_field = profile.get(f"{field}_commit") or ""
|
|
source_field = profile.get("source_commit") or ""
|
|
|
|
if not isinstance(raw, dict):
|
|
if not raw:
|
|
return []
|
|
return [
|
|
(
|
|
str(raw),
|
|
_by_mode(pin_field, label),
|
|
_by_mode(source_field, label),
|
|
)
|
|
]
|
|
|
|
# The requested build mode first, so it wins when several repositories
|
|
# would otherwise be equally good candidates.
|
|
ordered = ([label] if label and label in raw else []) + [
|
|
key for key in raw if key != label
|
|
]
|
|
return [
|
|
(str(raw[key]), _by_mode(pin_field, key), _by_mode(source_field, key))
|
|
for key in ordered
|
|
if raw.get(key)
|
|
]
|
|
|
|
# Forges the pages link into but upstream.py does not read from. Each is
|
|
# reached by browser only, so the Repo carries no API or raw base.
|
|
_BROWSE_ONLY = {
|
|
"voidpoint.io": "gitlab",
|
|
"bitbucket.org": "bitbucket",
|
|
"sourceforge.net": "sourceforge",
|
|
"archive.softwareheritage.org": "swh",
|
|
}
|
|
|
|
|
|
def _browse_repo(url: str) -> upstream.Repo | None:
|
|
"""The repository a profile URL names, on any forge the pages can link.
|
|
|
|
Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
|
|
Heritage showed no revision and no linked citation: parse_repo knows
|
|
only the forges profile_sync fetches from.
|
|
"""
|
|
repo = upstream.parse_repo(url)
|
|
if repo is not None or not url:
|
|
return repo
|
|
parts = urllib.parse.urlsplit(url.strip())
|
|
family = _BROWSE_ONLY.get(parts.netloc)
|
|
segments = [seg for seg in parts.path.split("/") if seg]
|
|
if family == "gitlab":
|
|
if "-" in segments:
|
|
segments = segments[: segments.index("-")]
|
|
if len(segments) < 2:
|
|
return None
|
|
owner, name = "/".join(segments[:-1]), segments[-1]
|
|
elif family == "bitbucket" and len(segments) >= 2:
|
|
owner, name = segments[0], segments[1]
|
|
elif family == "sourceforge" and len(segments) >= 3 and segments[0] == "p":
|
|
owner, name = segments[1], segments[2]
|
|
elif family == "swh":
|
|
# A revision URL names no project: the pin alone addresses the tree.
|
|
owner, name = "softwareheritage", "revision"
|
|
else:
|
|
return None
|
|
return upstream.Repo(parts.netloc, family, "", "", owner, name.removesuffix(".git"))
|
|
|
|
|
|
def _forge_sources(profile: dict, label: str = "") -> list[tuple[upstream.Repo, str]]:
|
|
"""Supported source repositories and immutable revisions for a profile."""
|
|
candidates: list[tuple[upstream.Repo, str]] = []
|
|
seen: set[tuple[str, str, str]] = set()
|
|
source_repos: set[tuple[str, str]] = set()
|
|
|
|
for field in ("source", "upstream"):
|
|
for value, pin, fallback in _repo_pins(profile, field, label):
|
|
repo = _browse_repo(value)
|
|
if repo is None:
|
|
continue
|
|
repo_key = (repo.host, repo.slug)
|
|
if field == "upstream" and not pin and repo_key in source_repos:
|
|
pin = fallback
|
|
if not pin:
|
|
continue
|
|
key = (repo.host, repo.slug, pin)
|
|
if key not in seen:
|
|
candidates.append((repo, pin))
|
|
seen.add(key)
|
|
if field == "source":
|
|
source_repos.add(repo_key)
|
|
return candidates
|
|
|
|
def _source_permalink(repo: upstream.Repo, pin: str, path: str,
|
|
start: int | None, end: int | None) -> str:
|
|
"""Forge-specific browser URL for one file at one immutable revision."""
|
|
quoted_path = urllib.parse.quote(path, safe="/@:+-._~")
|
|
base = f"https://{repo.host}/{repo.owner}/{repo.name}"
|
|
if repo.family == "bitbucket":
|
|
url = f"{base}/src/{pin}/{quoted_path}"
|
|
if start is not None:
|
|
url += f"#lines-{start}" + (f":{end}" if end is not None and end != start else "")
|
|
return url
|
|
if repo.family == "sourceforge":
|
|
# The Allura code browser takes no line range: the file at the pin.
|
|
return f"https://{repo.host}/p/{repo.owner}/{repo.name}/ci/{pin}/tree/{quoted_path}"
|
|
if repo.family == "swh":
|
|
url = f"https://{repo.host}/browse/revision/{pin}/?path={quoted_path}"
|
|
if start is not None:
|
|
url += f"#L{start}" + (f"-L{end}" if end is not None and end != start else "")
|
|
return url
|
|
if repo.family == "github":
|
|
url = f"{base}/blob/{pin}/{quoted_path}"
|
|
elif repo.family == "gitlab":
|
|
url = f"{base}/-/blob/{pin}/{quoted_path}"
|
|
else:
|
|
url = f"{base}/src/commit/{pin}/{quoted_path}"
|
|
if start is not None:
|
|
if repo.family == "gitlab":
|
|
url += f"#L{start}"
|
|
if end is not None and end != start:
|
|
url += f"-{end}"
|
|
else:
|
|
url += f"#L{start}"
|
|
if end is not None and end != start:
|
|
url += f"-L{end}"
|
|
return url
|
|
|
|
# The tree listings profile_sync caches. Read offline only: a page build
|
|
# never calls a forge, and a listing it does not find leaves the link as the
|
|
# profile's declarations alone decide it.
|
|
TREE_CACHE = Path(__file__).resolve().parent.parent / upstream.CACHE_DIR
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str] | None:
|
|
"""Every file of a repository at a revision, None when not cached or
|
|
truncated (a missing path would then prove nothing)."""
|
|
if not repo.api_base:
|
|
return None # a browse-only forge: no listing to read
|
|
paths, truncated = upstream.list_tree(repo, pin, str(TREE_CACHE), offline=True)
|
|
return frozenset(paths) if paths and not truncated else None
|
|
|
|
|
|
def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool | None:
|
|
"""Whether the repository holds ``path`` at ``pin``: None when its tree
|
|
is not cached. trident cites src/core/loader/ncch.cpp, a Panda3DS file,
|
|
and the page linked it into 3dsTrident, where it is a 404."""
|
|
paths = _tree_paths(repo, pin)
|
|
return None if paths is None else path in paths
|
|
|
|
|
|
def _holder(
|
|
candidates: list[tuple[upstream.Repo, str]], path: str, default: tuple | None,
|
|
) -> tuple[upstream.Repo, str] | None:
|
|
"""The pinned repository that holds ``path``, source before upstream as
|
|
profile_sync.select_views orders them. Without every tree cached, the
|
|
declarations decide (``default``). A path no cached tree holds links
|
|
nowhere: no link beats a link to a file that is not there."""
|
|
verdicts = [tree_holds(repo, pin, path) for repo, pin in candidates]
|
|
if not candidates or None in verdicts:
|
|
if default is not None and tree_holds(*default, path) is False:
|
|
return None
|
|
return default
|
|
return next((cand for cand, held in zip(candidates, verdicts) if held), None)
|
|
|
|
|
|
def _commit_url(repo: upstream.Repo, pin: str) -> str:
|
|
"""Forge-specific browser URL for one commit."""
|
|
base = f"https://{repo.host}/{repo.owner}/{repo.name}"
|
|
if repo.family == "gitlab":
|
|
return f"{base}/-/commit/{pin}"
|
|
if repo.family == "bitbucket":
|
|
return f"{base}/commits/{pin}"
|
|
if repo.family == "sourceforge":
|
|
return f"https://{repo.host}/p/{repo.owner}/{repo.name}/ci/{pin}/"
|
|
if repo.family == "swh":
|
|
return f"https://{repo.host}/browse/revision/{pin}/"
|
|
return f"{base}/commit/{pin}"
|
|
|
|
|
|
def revision_markdown(profile: dict) -> str:
|
|
"""Each repository the profile reads, linked at the commit it was read at.
|
|
|
|
The page linked the repositories unpinned: the revision every citation
|
|
refers to sat in the YAML alone, for 178 profiles.
|
|
"""
|
|
links = [
|
|
f"[{'Software Heritage' if repo.family == 'swh' else repo.slug}@{pin[:12]}]"
|
|
f"({_commit_url(repo, pin)})"
|
|
for repo, pin in _forge_sources(profile)
|
|
]
|
|
branch = profile.get("source_branch")
|
|
if branch and not isinstance(branch, dict):
|
|
links.append(f"branch `{branch}`")
|
|
elif isinstance(branch, dict):
|
|
links += [f"{mode} branch `{name}`" for mode, name in branch.items()]
|
|
return ", ".join(links)
|
|
|
|
|
|
_GLUE_RE = re.compile(r"(^|/)(libretro|retro_)[^/]*$|(^|/)libretro[^/]*/", re.IGNORECASE)
|
|
|
|
|
|
def role_label(profile: dict, role: str, path: str = "") -> str:
|
|
"""Whose code a cited path is, said the way a reader asks it.
|
|
|
|
A libretro port carries the emulator's own sources beside its glue: pcem
|
|
cites src/mem.c in the port, which is PCem's code, not the core's.
|
|
"""
|
|
if role == "upstream":
|
|
return "upstream emulator"
|
|
if "libretro" not in str(profile.get("type", "")):
|
|
return "this build"
|
|
if _GLUE_RE.search(path):
|
|
return "libretro glue"
|
|
return "emulator code, libretro port"
|
|
|
|
|
|
def role_sources(profile: dict, label: str = "") -> dict[str, tuple[upstream.Repo, str]]:
|
|
"""The forge repository and pin behind each role a citation can name."""
|
|
found: dict[str, tuple[upstream.Repo, str]] = {}
|
|
for field in ("source", "upstream"):
|
|
for value, pin, _fallback in _repo_pins(profile, field, label):
|
|
repo = _browse_repo(value)
|
|
if repo is not None and pin and field not in found:
|
|
found[field] = (repo, pin)
|
|
return found
|
|
|
|
|
|
def requested_by(profile: dict, value: str | dict | None) -> str:
|
|
"""Whose code the citations of an entry are in, said in words."""
|
|
labels = []
|
|
for _label, refs in source_ref_values(value):
|
|
for part in split_source_ref(refs):
|
|
role, bare = ref_role(part.path)
|
|
if role:
|
|
label = role_label(profile, role, bare)
|
|
if label not in labels:
|
|
labels.append(label)
|
|
if not labels or not _two_repositories(profile):
|
|
return ""
|
|
return labels[0] if len(labels) == 1 else ", ".join(labels[:-1]) + f" and {labels[-1]}"
|
|
|
|
|
|
def _two_repositories(profile: dict) -> bool:
|
|
source, upstream_url = profile.get("source"), profile.get("upstream")
|
|
return bool(source and upstream_url and source != upstream_url)
|
|
|
|
|
|
def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
|
|
"""Render source_ref values as pinned links when their forge is known.
|
|
|
|
A profile can declare two repositories: the libretro port in ``source`` and
|
|
the original emulator in ``upstream``. Which of the two carries a given
|
|
path cannot be decided without reading their trees, and this generator runs
|
|
offline. Rather than guess, an unattributable path is rendered as plain
|
|
code: a citation with no link still names the file and the lines, while a
|
|
link to the wrong repository is a false citation.
|
|
"""
|
|
rendered_groups: list[str] = []
|
|
path_re = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
|
|
for label, refs in source_ref_values(value):
|
|
candidates = _forge_sources(profile, label)
|
|
by_role = role_sources(profile, label)
|
|
rendered: list[str] = []
|
|
current_role = ""
|
|
for part in split_source_ref(refs):
|
|
role, bare = ref_role(part.path)
|
|
if bare != part.path or role:
|
|
current_role = role
|
|
elif part.raw and not part.raw[:1].isdigit():
|
|
current_role = ""
|
|
part = type(part)(bare, part.start, part.end, part.raw)
|
|
display = part.path
|
|
if part.start is not None:
|
|
display += f":{part.start}"
|
|
if part.end is not None and part.end != part.start:
|
|
display += f"-{part.end}"
|
|
|
|
# The path is linked as written. A first segment equal to a
|
|
# repository's name proves nothing: PCSX2 keeps its sources in
|
|
# pcsx2/, hatariB vendors Hatari in hatari/, and stripping it
|
|
# broke 470 links. Two repositories and no tree to read: no link.
|
|
selected: tuple[upstream.Repo, str] | None = None
|
|
real_path = part.path
|
|
if path_re.fullmatch(part.path):
|
|
if current_role:
|
|
# Only the repository the part names, or no link at all.
|
|
selected = by_role.get(current_role)
|
|
if selected is not None and tree_holds(*selected, part.path) is False:
|
|
selected = None
|
|
else:
|
|
selected = _holder(
|
|
candidates, part.path,
|
|
candidates[0] if len(candidates) == 1 else None,
|
|
)
|
|
|
|
tag = (f" ({role_label(profile, current_role, part.path)})"
|
|
if current_role and _two_repositories(profile) else "")
|
|
if selected is None:
|
|
rendered.append(f"`{display}`{tag}")
|
|
else:
|
|
repo, pin = selected
|
|
url = _source_permalink(
|
|
repo, pin, real_path, part.start, part.end
|
|
)
|
|
rendered.append(f"[`{display}`]({url}){tag}")
|
|
|
|
text = ", ".join(rendered) if rendered else f"`{refs}`"
|
|
if label:
|
|
text = f"**{label}:** {text}"
|
|
rendered_groups.append(text)
|
|
return ", ".join(rendered_groups)
|
|
|
|
|
|
def link_prose_citations(profile: dict, text: str) -> str:
|
|
"""Prose file:line citations as pinned links, where the target is certain.
|
|
|
|
profile_sync keeps these citations on their pin, and the page printed
|
|
4,328 of them as plain text. A citation links only when the profile reads
|
|
one repository and the path is rooted (it holds a directory) or is one a
|
|
structured source_ref of the profile already uses: a bare name could be
|
|
anywhere in the tree, and a link to the wrong file is a false citation.
|
|
Text already in backticks or in a link is left alone.
|
|
"""
|
|
candidates = _forge_sources(profile)
|
|
if len(candidates) != 1 or ":" not in text:
|
|
return text
|
|
repo, pin = candidates[0]
|
|
structured = {
|
|
part.path
|
|
for entry in profile.get("files") or []
|
|
if isinstance(entry, dict)
|
|
for _label, refs in source_ref_values(entry.get("source_ref"))
|
|
for part in split_source_ref(refs)
|
|
}
|
|
out = []
|
|
for line in text.split("\n"):
|
|
guarded = _guarded_spans(line)
|
|
edits = []
|
|
for _n, path_span, spans, parts, _run, wrapped in _prose_runs(line):
|
|
path = parts[0].path
|
|
if wrapped or " " in path or not _PATH_RE.fullmatch(path):
|
|
continue
|
|
if "/" not in path and path not in structured:
|
|
continue
|
|
role = parts[0].role
|
|
if role and (role_sources(profile).get(role) or (None,))[0] != repo:
|
|
# `upstream a.c:5` names a repository this link would not reach.
|
|
continue
|
|
if any(lo <= path_span[0] < hi for lo, hi in guarded):
|
|
continue
|
|
if tree_holds(repo, pin, path) is False:
|
|
continue
|
|
for index, ((lo, hi), part) in enumerate(zip(spans, parts)):
|
|
start = path_span[0] if index == 0 else lo
|
|
url = _source_permalink(repo, pin, part.path, part.start, part.end)
|
|
edits.append((start, hi, f"[`{line[start:hi]}`]({url})"))
|
|
for start, end, link in sorted(edits, reverse=True):
|
|
line = line[:start] + link + line[end:]
|
|
out.append(line)
|
|
return "\n".join(out)
|
|
|
|
|
|
_PATH_RE = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
|
|
|
|
|
|
def _guarded_spans(line: str) -> list[tuple[int, int]]:
|
|
"""Code spans and links of a markdown line."""
|
|
return [m.span() for m in re.finditer(r"`[^`]*`|\[[^\]]*\]\([^)]*\)", line)]
|
|
|
|
|
|
# Where a platform's list was read: the forge, the repository, the revision
|
|
# and the path inside it, out of the URL its scraper pinned.
|
|
_LIST_URL_RES = (
|
|
re.compile(r"^https://raw\.githubusercontent\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/"
|
|
r"(?P<rev>[^/]+)/(?P<path>.+)$"),
|
|
re.compile(r"^https://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/(?:blob|tree)/"
|
|
r"(?P<rev>[^/]+)(?:/(?P<path>.+))?$"),
|
|
re.compile(r"^https://codeload\.github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/"
|
|
r"tar\.gz/(?P<rev>[^/]+)$"),
|
|
re.compile(r"^https://(?P<host>gitlab\.com)/(?P<owner>[^/]+)/(?P<repo>[^/]+)/-/"
|
|
r"(?:raw|blob)/(?P<rev>[^/]+)/(?P<path>.+)$"),
|
|
)
|
|
_BRANCHES = frozenset({"master", "main", "db"})
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ListSource:
|
|
"""A platform's list as its scraper read it."""
|
|
|
|
host: str
|
|
owner: str
|
|
repo: str
|
|
rev: str
|
|
path: str
|
|
|
|
@property
|
|
def moving(self) -> bool:
|
|
"""The revision is a branch: what was read is not recorded."""
|
|
return not self.rev or self.rev in _BRANCHES
|
|
|
|
@property
|
|
def label(self) -> str:
|
|
"""`batocera-systems at tag batocera-43.1`, said in words."""
|
|
name = self.path.rsplit("/", 1)[-1] if self.path else f"{self.repo} repository"
|
|
if not self.rev:
|
|
return f"{name}, default branch"
|
|
if re.fullmatch(r"[0-9a-f]{40}", self.rev):
|
|
return f"{name} at commit {self.rev[:12]}"
|
|
if self.moving:
|
|
return f"{name} on branch {self.rev}"
|
|
return f"{name} at tag {self.rev}"
|
|
|
|
def browse(self, path: str = "", start: int | None = None,
|
|
end: int | None = None) -> str:
|
|
"""The page a reader opens: the file, or the tree, at the revision."""
|
|
path = path or self.path
|
|
if self.host == "gitlab.com":
|
|
url = f"https://gitlab.com/{self.owner}/{self.repo}/-/blob/{self.rev}/{path}"
|
|
anchor = f"#L{start}" + (f"-{end}" if end and end != start else "")
|
|
else:
|
|
base = f"https://github.com/{self.owner}/{self.repo}"
|
|
if not self.rev:
|
|
return base
|
|
kind = "blob" if path else "tree"
|
|
url = f"{base}/{kind}/{self.rev}" + (f"/{path}" if path else "")
|
|
anchor = f"#L{start}" + (f"-L{end}" if end and end != start else "")
|
|
return url + (anchor if start else "")
|
|
|
|
|
|
def list_source(url: str) -> ListSource | None:
|
|
"""The forge, repository, revision and path of a platform's `source:`."""
|
|
url = str(url or "").strip()
|
|
for pattern in _LIST_URL_RES:
|
|
found = pattern.match(url)
|
|
if found:
|
|
parts = found.groupdict()
|
|
return ListSource(parts.get("host") or "github.com", parts["owner"],
|
|
parts["repo"], parts["rev"], parts.get("path") or "")
|
|
bare = re.match(r"^https://github\.com/([^/]+)/([^/]+?)/?$", url)
|
|
if bare:
|
|
return ListSource("github.com", bare[1], bare[2], "", "")
|
|
return None
|
|
|
|
|
|
def launcher_ref_markdown(source: ListSource | None, ref: str) -> str:
|
|
"""A launcher record's source_ref, each part a pinned link.
|
|
|
|
Linked only where the caller knows the launcher code sits in the
|
|
repository and revision of the platform's list. Otherwise each part is
|
|
plain code: a link to another repository is a false citation.
|
|
"""
|
|
rendered: list[str] = []
|
|
for part in split_source_ref(ref):
|
|
display = part.path.rsplit("/", 1)[-1]
|
|
if part.start is not None:
|
|
display += f":{part.start}"
|
|
if part.end is not None and part.end != part.start:
|
|
display += f"-{part.end}"
|
|
if source is None or not source.rev or not _PATH_RE.fullmatch(part.path):
|
|
rendered.append(f"`{display}`")
|
|
else:
|
|
rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})")
|
|
return ", ".join(rendered)
|