kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-12 04:53:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge origin/main

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 00:07:47 +01:00
2 parents d265453b32 9a66ac3219
commit 0983ae0188
40 files changed
+1304 -222
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/cross_reference.py
+16 -6
View File
@@ -27,6 +27,7 @@ from common import (
platform_arg,
ProfileSelectionError,
_norm_system_id,
archive_entry,
directory_members,
expand_directory_entries,
get_mame_clone_map,
@@ -573,18 +574,27 @@ class FileConsumers:
for dest, sha1 in members.items()
]
else:
# The archive members the profile reads decide between
# same-named copies, as in the emulator pack and verify.
found = [self.sha1_of(
with_archive_members(entry, files), entry.get("path") or "",
owner=name,
)]
found = [self.sha1_of(*entry_target(entry, files), owner=name)]
for sha1 in found:
if sha1:
self.emulators.setdefault(sha1, set()).add(name)
self.emulator_entries.setdefault(sha1, {}).setdefault(name, index)
def entry_target(entry: dict, files: list[dict]) -> tuple[dict, str]:
"""The entry a profile entry resolves through, and its destination.
A ROM read inside an archive is the archive: FBNeo opens sm1.sm1 in
neogeo.zip, and resolved by its crc32 it credited a loose copy FBNeo
never opens, while no neogeo.zip card named FBNeo. The archive
members the profile reads decide between same-named copies, as in
the emulator pack and verify.
"""
if entry.get("archive"):
return archive_entry(entry["archive"], files), ""
return with_archive_members(entry, files), entry.get("path") or ""
def entry_source(f: dict, index: dict) -> str | None:
"""Where the collection holds a profile entry, or None.
scripts/generate_readme.py
+14 -5
View File
@@ -37,6 +37,7 @@ from common import (
from nativemode import check_label
import release_record
from siteguides import install_command
from validate_pr import LARGE_FILE_ISSUE_URL
from verify import verify_platform
@@ -929,8 +930,12 @@ the most useful contribution.
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
`python scripts/validate_pr.py path/to/file` names that folder for a file still
outside `bios/`. Given a folder, such as an old card's BIOS folder, it checks
every file under it. A file over the browser's 25 MB limit goes in that way.
outside `bios/`, and `--place` copies it there under its declared name. Given a
folder, such as an old card's BIOS folder, it checks every file under it. A
file over the browser's 25 MB limit goes in that way, up to the 100 MB GitHub
accepts in a push. A larger file stays out of the pull request: [open an
issue]({LARGE_FILE_ISSUE_URL}) with the hashes `validate_pr.py` prints, and a
maintainer adds it to the `large-files` release.
## Add a platform
@@ -969,18 +974,22 @@ validation above is its check. For code, profiles or platform files, these are
the checks CI runs on a pull request (Python 3.12 or newer):
```bash
pip install pyyaml "jsonschema[format-nongpl]" markdown
pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin
python scripts/validate_pr.py --changed # the BIOS files the branch adds
python scripts/validate_schemas.py --source-only # profiles and platform files
python scripts/profile_sync.py --added --base upstream/main --strict # each profile the branch adds
python scripts/refresh_data_dirs.py # data directories the pack-building tests read
python scripts/generate_site.py # the pages the site tests read
npm ci --prefix website && npm run build --prefix website # the built site, Node 24
python -m unittest discover -s tests -t .
python scripts/validate_site.py --site-dir website/dist # links and page budgets
vermin --no-tips -t=3.8- --violations install.py # install.py stays on Python 3.8
```
`upstream` is this repository as a remote of your fork
(`git remote add upstream https://github.com/Abdess/retrobios`). A profile you
only edit is not held to `--strict`: its older refs are reviewed by hand.
(`git remote add upstream https://github.com/Abdess/retrobios`, then
`git fetch upstream`). A profile you only edit is not held to `--strict`: its
older refs are reviewed by hand.
The full pipeline is not needed for a contribution: it rebuilds every pack
in `dist/`, which is why maintainers run it before a release.
scripts/generate_site.py
+94 -57
View File
@@ -63,7 +63,7 @@ from common import (
)
from artifacts import ArtifactLockBusy, artifact_lock
from artifacts import json_text as _json_text
from cross_reference import FileConsumers, held_copy
from cross_reference import FileConsumers, entry_target, held_copy
from nativemode import check_label, digest_algorithm, reads_file_contents
@@ -679,9 +679,13 @@ def release_line(platform_name: str, record: dict) -> list[str]:
return lines
# What a reader on another version of the setup does with the pack. Our data
# says which list the pack was made from, not what an older list lacks.
OTHER_VERSION = "Other version? Run `--check` after copying."
def other_version(platform_name: str, prefix: str = "../") -> str:
"""What a reader on another version of the setup does with the pack.
Our data says which list the pack was made from, not what an older list
lacks. A bare `--check` was no command to run for someone holding a ZIP:
the line links the setup page's commands that check."""
target = f"{prefix}platforms/{platform_name}.md#how-to-check"
return f"Other version? [Check the files]({target}) after copying."
def setup_version(platform_name: str, record: dict | None = None) -> str:
@@ -700,13 +704,15 @@ def setup_version(platform_name: str, record: dict | None = None) -> str:
return f"{config.get('platform', platform_name)} {version.removeprefix('v')}"
def version_line(platform_name: str, record: dict, named: bool = False) -> str:
def version_line(platform_name: str, record: dict, named: bool = False,
prefix: str = "../") -> str:
"""The version the pack fits, and what to do on another. *named* when
the release line already named the pack's version."""
made_from = setup_version(platform_name, record)
other = other_version(platform_name, prefix)
if named or not made_from:
return OTHER_VERSION
return f"Made for {made_from}. {OTHER_VERSION}"
return other
return f"Made for {made_from}. {other}"
# Why a setup has no ZIP, answered once on the FAQ.
@@ -793,7 +799,8 @@ def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../",
)
released = release_line(platform_name, record)
return "\n\n".join([buttons, *released,
version_line(platform_name, record, named=len(released) > 1)])
version_line(platform_name, record, named=len(released) > 1,
prefix=prefix)])
def generate_get_page(platform_name: str) -> str:
@@ -2191,9 +2198,11 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
"when one is known, the `system_name` the site gives its system. "
"[Find or check a file](files/index.md) reads it.",
"",
"In a browser, the SHA1 of a file the reader picks finds its records. Web "
"Crypto has no MD5, and none is needed: a setup that lists a file by MD5 "
"or CRC32 is filed under the SHA1 of the collection file it names too.",
"In a browser, the SHA1 of a file the reader picks finds its records when "
"the collection holds that file. A list naming a file the collection lacks "
"is filed under its MD5 or CRC32 alone, and Web Crypto has no MD5: "
"that page computes MD5 and CRC32 in script and looks "
"up all four.",
"",
"```js",
"// The records of the file a reader picked, by its SHA1.",
@@ -2443,7 +2452,7 @@ def _browser_title(relative: Path, title: str) -> str:
# The words a web search for a console or an emulator types: "dreamcast
# bios", "duckstation bios". The h1 and the sidebar keep the name.
if key.startswith("emulators/"):
has_files = PAGE_DATA.get(key, {}).get("hasPart")
has_files = PAGE_DATA.get(key, {}).get("hasPart") and key not in NOTHING_TO_FETCH
# Lexaloffle, QEMU and ScummVM name a system page too.
own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else ""
return f"{title}{own} BIOS files" if has_files else f"{title} emulator"
@@ -2514,6 +2523,12 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
return _system_description(markdown, browser)
if key.startswith("emulators/") and relative.stem != "index":
return _emulator_description(markdown, key, title, _first_paragraph(lead))
if key.startswith("platforms/") and relative.stem != "index":
# The summary ends on a link to the platform's site, whose label
# alone was left: "A system for handhelds. ROCKNIX website."
summary = _first_paragraph(lead).removesuffix(f" {title} website.")
return _shorten(f"The BIOS files for {title}, the folder they go in and the "
f"command that installs them. {summary}", 155)
first = _first_paragraph(lead, minimum=35)
questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading))
for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)]
@@ -2539,7 +2554,9 @@ def _emulator_description(markdown: str, key: str, title: str, intro: str) -> st
its profile table when the profile names no system."""
notes = re.search(r"<summary>Technical notes</summary>\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S)
if not intro and notes:
sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0]
# An initial is no sentence end: hcl's read "open-source port of E."
sentence = re.split(r"(?<![\s(][A-Z]\.)(?<=\.)\s", _plain_markdown(notes[1]),
maxsplit=1)[0]
intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}"
if not intro:
row = {name: _plain_markdown(value) for name, value in re.findall(
@@ -2553,6 +2570,8 @@ def _emulator_description(markdown: str, key: str, title: str, intro: str) -> st
elif title not in intro:
# "It reads 1 file": the profile names no system to open on.
intro = f"{title}: {intro[:1].lower()}{intro[1:]}"
if key in NOTHING_TO_FETCH:
return _shorten(f"{intro} Every file it reads comes with the app.", 155)
# An alias page sends the reader to the core whose files it shares.
if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro:
return _shorten(intro, 155)
@@ -2623,6 +2642,9 @@ def _system_description(markdown: str, browser: str) -> str:
# Structured data a page generator knows beyond its text, by page path
# relative to docs/: read by decorate_markdown_pages.
PAGE_DATA: dict[str, dict] = {}
# Emulator pages whose every file comes with the app: no "BIOS files" in
# their title, and their description says so (DaedalusX64, ZEsarUX).
NOTHING_TO_FETCH: set[str] = set()
# Pages that hold part of a page too long for one (a console of a split
# manufacturer, a run of an emulator's cards), by path relative to docs/:
# their H1 already says what they are.
@@ -2950,6 +2972,17 @@ def release_versions(profile: dict) -> dict[str, str]:
if re.search(r"[0-9]", value) and not names_a_commit(value)}
def record_emulator_page(name: str, profile: dict) -> None:
"""What the title and the description of an emulator page read."""
key = f"emulators/{name}.md"
PAGE_DATA[key] = emulator_structured_data(profile)
listed = [f for f in profile.get("files") or [] if isinstance(f, dict)]
if listed and all(comes_with_app(f) for f in listed):
NOTHING_TO_FETCH.add(key)
else:
NOTHING_TO_FETCH.discard(key)
def emulator_structured_data(profile: dict) -> dict:
"""What an emulator page states, for a reader of its JSON-LD.
@@ -4176,7 +4209,7 @@ def _setup_routes(name: str, display: str, page_link: bool = False) -> list[str]
# lines are on the get/ page, the one the command is shared from.
tabs = ("windows",) if page_link else ("windows", "macos", "linux")
computer += [
"Needs the device on your network. Run the command on the computer.",
"Needs the device on your network.",
"",
*terminal_tabs(tabs, args, how=how),
*([f"[Mac or Linux](../{get_page_link(name)}#from-your-computer)", ""]
@@ -6443,13 +6476,6 @@ def _region_order() -> dict[str, int]:
return {region: index for index, region in enumerate(ordered)}
_REGION_WORDS = {"uk": "UK", "asia-ntsc": "Asia (NTSC)", "asia-pal": "Asia (PAL)"}
def _region_word(region: str) -> str:
return _REGION_WORDS.get(region) or region.replace("-", " ").title()
def _reading_pairs(sha1: str, consumers: FileConsumers) -> list[tuple[str, dict]]:
"""The profile entries that read a file, each with its emulator."""
files_of = getattr(consumers, "emulator_files", {})
@@ -6662,7 +6688,7 @@ def _start_groups(
for f in primaries:
sha1 = str(f.get("sha1", ""))
for emulator, entry in _reading_pairs(sha1, consumers):
if entry.get("bundled"):
if comes_with_app(entry):
continue
system = None
systems = _entry_systems(emulator, entry)
@@ -6677,7 +6703,7 @@ def _start_groups(
for sha1, emulator, entry, system in _entries_by_section(consumers).get(here, []):
record = known.get(sha1)
parts = str((record or {}).get("path", "")).split("/")
if (sha1 in held or not record or entry.get("bundled") or ".variants" in parts
if (sha1 in held or not record or comes_with_app(entry) or ".variants" in parts
or len(parts) < 3 or parts[1] != "Arcade"):
continue
if sha1 not in kept:
@@ -7031,7 +7057,7 @@ def _group_start(group: _StartGroup, first_below: bool) -> list[tuple[str, str]]
else:
regional[(*same, region)] = regional.pop(same)
out = [(", ".join(_region_word(r) for r in regions), regional[regions])
out = [(", ".join(_region_words(r) for r in regions), regional[regions])
for regions in sorted(regional, key=lambda rs: (order.get(rs[0], len(order)), rs))]
# A name a region line gives already is no "also", and one only
# emulators no region line answers reads is an "instead": armsx1 reads
@@ -7819,12 +7845,15 @@ def generate_emulators_index(profiles: dict) -> str:
def comes_with_app(f: dict) -> bool:
"""The emulator installs the file in every build the profile documents.
"""The emulator installs the file in the build the entry describes.
``bundled`` narrowed by ``mode`` is installed by one build only, and the
other build still reads it from the user's folder.
The rule verify and the pack read (``installed_with_emulator``). An entry
narrowed by ``mode`` is that build's alone: the other build reads its
own entry, or nothing.
"""
return f.get("bundled") is True and not f.get("mode")
from validation import installed_with_emulator # noqa: PLC0415
return installed_with_emulator(f)
def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]:
@@ -7841,7 +7870,6 @@ def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]:
category = f.get("category", "")
region = f.get("region", "")
storage = f.get("storage", "")
bundled = f.get("bundled", False)
embedded = f.get("embedded", False)
has_builtin = f.get("has_builtin", False)
archive = f.get("archive", "")
@@ -7851,9 +7879,10 @@ def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]:
if shipped:
# NetherSX2's three interface fonts sat beside the BIOS as
# "required" and "missing": the app ships them, nobody fetches them.
badges.append(
'<span class="rb-badge rb-badge-info">comes with the app</span>'
)
# Narrowed to one build, the build is named.
build = f.get("mode") if f.get("mode") in ("standalone", "libretro") else ""
label = f"installed with the {build} emulator" if build else "comes with the app"
badges.append(f'<span class="rb-badge rb-badge-info">{label}</span>')
elif required:
needed = f"required for {html.escape(for_system)}" if for_system else "required"
badges.append(
@@ -7879,7 +7908,7 @@ def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]:
badges.append(
'<span class="rb-badge rb-badge-info">HLE fallback</span>'
)
if mode:
if mode and not (shipped and mode in ("standalone", "libretro")):
badges.append(
f'<span class="rb-badge rb-badge-muted">{mode}</span>'
)
@@ -7896,13 +7925,6 @@ def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]:
badges.append(
f'<span class="rb-badge rb-badge-muted">{storage}</span>'
)
if bundled and not shipped:
# Said in words: the pack leaves the file out because the emulator,
# in the build the entry names, already installs it.
build = f" {f['mode']}" if f.get("mode") in ("standalone", "libretro") else ""
badges.append(
f'<span class="rb-badge rb-badge-muted">installed with the{build} emulator</span>'
)
if embedded:
badges.append(
'<span class="rb-badge rb-badge-muted">embedded</span>'
@@ -7995,10 +8017,14 @@ def _build_rows(profile: dict) -> list[str]:
found = _found_by_text(profile.get("found_by"))
if found:
rows.append(f"| Files found by | {found} |")
frozen = profile.get("pin_frozen")
if frozen:
held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|')
rows.append(f"| Revision held | {held} |")
# Each pin says why it is held: a bare upstream revision read as a
# profile left behind HEAD (mupen64plus, aps3e).
for field, label in (("pin_frozen", "Revision held"),
("upstream_frozen", "Upstream revision held")):
frozen = profile.get(field)
if frozen:
held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|')
rows.append(f"| {label} | {held} |")
announced = profile.get("version_source")
if isinstance(announced, dict) and announced.get("url"):
url = str(announced["url"])
@@ -8193,10 +8219,13 @@ def _render_emulator_file(
if consumers and f.get("type") == "directory":
details.extend(_directory_copies(f, consumers))
if consumers:
sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner)
sha1 = consumers.sha1_of(
*entry_target(f, profile.get("files") or []), owner=owner)
held = consumers.path_of(sha1) if sha1 and in_repo else ""
parts = held.split("/")
if len(parts) >= 3 and parts[0] == "bios":
if len(parts) >= 3 and parts[0] == "bios" and f.get("archive"):
details.append(_held_copy_line((held, f["name"]), consumers))
elif len(parts) >= 3 and parts[0] == "bios":
# Which copy answers this entry, linked to its collection card
# (provenance, every hash, download): an entry declaring no hash
# said "in repo" without naming the file.
@@ -8473,8 +8502,9 @@ def emulator_romset(profile: dict) -> str:
return ""
def _version_said(value: object, basis: dict) -> str:
"""One build's version and where the profile reads it."""
def _version_said(value: object, basis: dict, profile: dict, mode: str = "") -> str:
"""One build's version and where the profile reads it, linked to the
file or the tag where the source allows."""
kind = basis.get("kind")
if kind == "none":
return "none stated in the source"
@@ -8483,13 +8513,18 @@ def _version_said(value: object, basis: dict) -> str:
return f"unknown: {_UNVERIFIED.get(str(basis.get('why')), 'unread')}"
return "none published"
value = str(value)
url = version_basis_url(profile, basis, mode) if kind in ("file", "tag") else ""
def shown(text: object) -> str:
return f"[{_code(str(text))}]({url})" if url else _code(str(text))
if kind == "file":
return f"{value}, stated in `{basis.get('path')}`"
return f"{value}, stated in {shown(basis.get('path'))}"
if kind == "tag":
distance = int(basis.get("distance") or 0)
if not distance:
return f"{value}, the tag at this revision"
return f"{value}, {distance} commit{'s' if distance != 1 else ''} after tag `{basis.get('tag')}`"
return f"{value}, {distance} commit{'s' if distance != 1 else ''} after tag {shown(basis.get('tag'))}"
if kind == "package":
return f"{value}, read from {basis.get('artifact')}"
if kind == "unverified":
@@ -8524,10 +8559,10 @@ def version_markdown(profile: dict) -> str:
values = value if isinstance(value, dict) else {}
modes = list(dict.fromkeys([*values, *basis]))
return ", ".join(
f"{_version_said(values.get(mode), basis.get(mode) or {})} ({mode})"
f"{_version_said(values.get(mode), basis.get(mode) or {}, profile, mode)} ({mode})"
for mode in modes
)
return _version_said(value, basis)
return _version_said(value, basis, profile)
ROMSET_FAQ_PAGE = Path(WIKI_SRC_DIR) / "technical-faq.md"
@@ -8860,7 +8895,7 @@ def standalone_place(name: str, profile: dict, entry: dict, key: str = "") -> st
if not isinstance(entry, dict) or not profile.get("data_root"):
return ""
mode = entry.get("mode")
if mode == "libretro" or (entry.get("bundled") is True and mode in (None, "standalone")):
if mode == "libretro" or comes_with_app(entry):
return ""
excluded = _folder_only(name, entry)
if "*" in excluded or (key in excluded if key else excluded):
@@ -8898,7 +8933,7 @@ def _data_root_folder(name: str, profile: dict, key: str, entries: list[dict]) -
folders = set()
for f in entries:
mode = f.get("mode")
if mode == "libretro" or (f.get("bundled") is True and mode in (None, "standalone")):
if mode == "libretro" or comes_with_app(f):
continue
place = standalone_place(name, profile, f, key)
if not place:
@@ -12142,15 +12177,16 @@ def _generate(args: argparse.Namespace) -> None:
# The test each emulator page builds for itself: the parts a page is
# cut into weigh the status its rows show.
file_held = _availability_check(db, suppl_names)
# The plan names systems: the sections must be this run's data first.
manufacturers = _group_by_manufacturer(db)
set_console_sections(manufacturers, program_names(profiles))
plan_emulator_pages(profiles, file_held)
# Build system_id -> manufacturer page map (needed by all generators)
print("Building system cross-reference map...")
manufacturers = _group_by_manufacturer(db)
set_system_page_map(
_system_pages_from_data(manufacturers, coverages, db, profiles)
)
print(f" {len(_system_page_map)} system IDs mapped to pages")
set_console_sections(manufacturers, program_names(profiles))
from siterender import link_coverage, system_ids # noqa: PLC0415
print("Generating system pages...")
@@ -12300,7 +12336,7 @@ def _generate(args: argparse.Namespace) -> None:
name, profile, db, consumers, suppl_names, data_registry,
data_dirs=page_data_directories(name, profiles),
)
PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile)
record_emulator_page(name, profile)
# A frozen copy answers after the emulator: a frozen_snapshot, and a
# profile pinned to the revision one platform builds (pin_frozen).
# Reicast and Flycast 2021 LE answered "dc bios missing" before Flycast.
@@ -12466,6 +12502,7 @@ from sitesources import ( # noqa: E402,F401
_forge_sources,
_source_permalink,
_source_ref_markdown,
version_basis_url,
)
from siterender import ( # noqa: E402,F401
_by_mode,
scripts/pipeline.py
+7 -7
View File
@@ -36,8 +36,8 @@ come from .cache/large/ only.
Usage:
python scripts/pipeline.py # active platforms
python scripts/pipeline.py --include-archived # all platforms
python scripts/pipeline.py --skip-packs # steps 0 to 3b, 4d, 7 and 8
python scripts/pipeline.py --skip-docs # skip steps 7 and 8
python scripts/pipeline.py --skip-packs # skip steps 4, 4b, 4c, 5 and 6
python scripts/pipeline.py --skip-docs # skip steps 7, 8 and 8b
python scripts/pipeline.py --offline # no network, see above
"""
@@ -902,13 +902,13 @@ def main() -> None:
parser.add_argument(
"--skip-packs",
action="store_true",
help="Skip steps 4 to 6 except 4d: packs, manifests and their checks. "
"Steps 7 and 8 still rewrite README.md and the site pages in website/, "
"add --skip-docs to "
"stop after verify",
help="Skip steps 4, 4b, 4c, 5 and 6: packs, manifests and their checks. "
"Step 4d still gates the run, step 4e still rewrites install/games/, and "
"steps 7 to 8b rewrite README.md and the site unless --skip-docs is given",
)
parser.add_argument(
"--skip-docs", action="store_true", help="Skip README and site generation"
"--skip-docs", action="store_true",
help="Skip steps 7, 8 and 8b: README, installer page and site"
)
parser.add_argument(
"--offline", action="store_true",
scripts/profile_sync.py
+4
View File
@@ -4378,6 +4378,10 @@ def branch_profiles(emulators_dir: str, base: str, added_only: bool) -> list[str
branch point are not counted, and ends at the working tree, so a
profile not yet committed is checked too. A rename reads as an addition.
"""
if subprocess.run(["git", "rev-parse", "--verify", "--quiet", base],
capture_output=True, check=False).returncode:
remote = base.split("/", 1)[0] if "/" in base else "upstream"
raise SystemExit(f"ERROR: {base} is not in this clone: run git fetch {remote} first")
merge_base = _git_lines(["merge-base", base, "HEAD"])[0]
paths = _git_lines([
"diff", "--name-only", "--no-renames",
scripts/siteguides.py
+6
View File
@@ -454,6 +454,10 @@ def fence(shell: str, command: str, indent: str = "") -> list[str]:
return [f'{indent}```{shell} title="{FENCE_TITLES[shell]}"', f"{indent}{command}", f"{indent}```"]
# Terminals named by where the command runs, not by the system it runs on.
PLACES = ("device", "server")
def terminal_tabs(
tabs: tuple[str, ...], args: dict[str, str], indent: str = "", steps: bool = True,
how: dict[str, str] | None = None,
@@ -465,6 +469,8 @@ def terminal_tabs(
items = []
for key in tabs:
label, shared, shell = TERMINALS[key]
if len(tabs) == 1 and key in PLACES:
label = "" # one place, which the text around already names
text = (how or {}).get(key) or shared
body = [text, ""] if steps else []
items.append((label, body + fence(shell, install_command(shell, args.get(key, "")))))
scripts/sitemarkup.py
+10 -1
View File
@@ -139,8 +139,17 @@ def details(kind: str, title: str, body: list[str], indent: str = "", opened: bo
def tabs(items: list[tuple[str, list[str]]], indent: str = "", sync: str = "os") -> list[str]:
"""Content tabs. Tabs that share a label share the reader's choice
across pages."""
across pages.
One item is no choice: 142 bars offered a single tab, most of them
"On the device" under a heading that says so. Its body stands alone,
led by its label when it has one: the RetroPie card is read from Linux
only, and the label is where the page said so."""
lines: list[str] = []
if len(items) == 1:
label, body = items[0]
lead = [f"{indent}**{label}**", ""] if label else []
return [*lead, *_indented(_trim(body), indent), ""]
if _engine == "mkdocs":
for label, body in items:
lines += [f'{indent}=== "{label}"', "", *_indented(_trim(body), indent + " "), ""]
scripts/sitesources.py
+33 -2
View File
@@ -240,8 +240,15 @@ def _walk(value: object) -> Iterator[tuple[str, object]]:
def cited_paths(profile: dict) -> set[str]:
"""Every path a page of the profile could link: the parts of each
source_ref and the file:line citations of its prose."""
paths: set[str] = set()
source_ref, the file:line citations of its prose and the file its
version is read from."""
basis = profile.get("version_basis")
bases = ([basis] if isinstance(basis, dict) and "kind" in basis
else list(basis.values()) if isinstance(basis, dict) else [])
paths: set[str] = {
str(b["path"]) for b in bases
if isinstance(b, dict) and b.get("kind") == "file" and b.get("path")
}
for key, value in _walk(profile):
if key == "source_ref" and isinstance(value, (str, dict)):
for _label, refs in source_ref_values(value):
@@ -355,6 +362,30 @@ def _commit_url(repo: upstream.Repo, pin: str) -> str:
return f"{base}/commit/{pin}"
def version_basis_url(profile: dict, basis: dict, mode: str = "") -> str:
"""Where a reader checks a version: the file stating it at the pin, or
the tag it counts from. "" when neither can be linked.
The Version cell printed `libretro/handy.h` as bare text beside a
pinned Revision link, so the #define it reads was one search away.
"""
field = "upstream" if basis.get("repo") == "upstream" else "source"
declared = next(iter(_repo_pins(profile, field, mode)), None)
repo = _browse_repo(declared[0]) if declared else None
if repo is None:
return ""
kind, path, tag = basis.get("kind"), str(basis.get("path") or ""), str(basis.get("tag") or "")
if kind == "file" and path:
pin = next((p for r, p in _forge_sources(profile, mode)
if (r.host, r.slug) == (repo.host, repo.slug)), "")
return _source_permalink(repo, pin, path, None, None) if pin and tree_holds(
repo, pin, path) else ""
if kind == "tag" and tag and repo.family in ("github", "gitlab"):
tree = "/-/tree/" if repo.family == "gitlab" else "/tree/"
return f"https://{repo.host}/{repo.owner}/{repo.name}{tree}{urllib.parse.quote(tag)}"
return ""
def revision_markdown(profile: dict) -> str:
"""Each repository the profile reads, linked at the commit it was read at.
scripts/sitestarlight.py
+12 -1
View File
@@ -244,10 +244,21 @@ def _tooltip_line(line: str, pattern: re.Pattern, pending: dict[str, str]) -> st
return "".join(out)
_NAME_BEFORE = re.compile(r"(?:/|\b[A-Z][\w'-]*[ \t]+)$")
def _in_proper_name(term: str, before: str) -> bool:
"""A capitalised term after a capitalised word or a slash is part of a
name: Free Unix Spectrum Emulator, Windows Terminal, Documents/Dolphin
Emulator. An acronym (BIOS, ZIP) is the term wherever it stands."""
first = term.split()[0]
return first[:1].isupper() and not first.isupper() and bool(_NAME_BEFORE.search(before))
def _tooltip_text(text: str, pattern: re.Pattern, pending: dict[str, str]) -> str:
def swap(match: re.Match) -> str:
term = match.group(1)
if term not in pending:
if term not in pending or _in_proper_name(term, text[:match.start()]):
return term
definition = pending.pop(term)
return f'<abbr class="rb-term" title="{html.escape(definition)}">{term}</abbr>'
scripts/validate_pr.py
+107 -17
View File
@@ -21,6 +21,7 @@ import argparse
import json
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
@@ -36,6 +37,7 @@ from common import (
)
from hashing import parse_hash_list
from nativemode import digest_algorithm
from dedup import _in_nodedup_dir
from validation import frontend_digest_matches, inner_rom_check
try:
@@ -79,6 +81,13 @@ class ValidationResult:
self.md5 = ""
self.crc32 = ""
self.size = 0
# "held" (a copy of a file the collection has) or "unknown" (no list
# or profile declares it): no check fails, and neither is a file
# the pull request adds to the collection.
self.outcome = ""
# Where --place copies the file: one folder and one declared name, or
# the .variants/ slot of a name the collection already holds.
self.destination = ""
def add_check(self, passed: bool, message: str) -> None:
self.checks.append(("PASS" if passed else "FAIL", message))
@@ -93,8 +102,12 @@ class ValidationResult:
def passed(self) -> bool:
return all(s != "FAIL" for s, _ in self.checks)
@property
def status(self) -> str:
return "FAIL" if not self.passed else "REVIEW" if self.outcome else "PASS"
def to_markdown(self) -> str:
status = "PASS" if self.passed else "FAIL"
status = self.status
lines = [f"### {status} `{self.filename}`"]
lines.append("")
lines.append(f"- **Path**: `{self.filepath}`")
@@ -147,7 +160,7 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
under, read from the other files of that system it holds.
"""
known = {"sha1": set(), "md5": set(), "sha256": set(), "crc32": set(),
"names": set(), "folders": {}, "sizes": {}, "entries": {}}
"names": set(), "folders": {}, "sizes": {}, "entries": {}, "named": {}}
systems_of: dict[str, set[str]] = {}
folders_of: dict[str, dict[str, int]] = {}
files = (db or {}).get("files", {})
@@ -194,6 +207,9 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
)
for value in (*sha1s, *md5s):
systems_of.setdefault(value, set()).add(system_id)
for value in (*sha1s, *md5s, *sha256s):
if "name" in file_entry:
known["named"].setdefault(value, set()).add(file_entry["name"])
_note_folder(folders_of, system_id, files, by_md5, sha1s, md5s)
known["folders"] = _system_homes(systems_of, folders_of)
@@ -251,6 +267,7 @@ def load_profile_hashes(
"""
known: dict[str, dict] = {
"sha1": {}, "sha256": {}, "md5": {}, "crc32": {}, "sizes": {}, "folders": {},
"named": {},
}
systems_of: dict[str, set[str]] = {}
folders_of: dict[str, dict[str, int]] = {}
@@ -274,6 +291,8 @@ def load_profile_hashes(
for algo, values in hashes.items():
for value in values:
known[algo].setdefault(value, set()).add(emulator)
if isinstance(entry.get("name"), str):
known["named"].setdefault(value, set()).add(entry["name"])
# The entry's own system, or the profile's when it emulates one.
system = entry.get("system") or (
declared_systems[0] if len(declared_systems) == 1 else None
@@ -374,9 +393,15 @@ def validate_file(
# PR that regenerated database.json was told each new file
# duplicated itself.
result.add_check(True, "Already in the database at this path")
elif _in_nodedup_dir(relative):
result.add_info(
f"Identical to `{existing['path']}`, kept: this tree is read by path"
)
else:
result.outcome = "held"
result.add_warning(
f"Duplicate: identical file already exists at `{existing['path']}`"
f"Already held at `{existing['path']}`: nothing to add, "
"leave this copy out of the pull request"
)
else:
result.add_check(True, "Not a duplicate in database")
@@ -421,8 +446,11 @@ def validate_file(
"Filename matches a known requirement but hash differs - may be a variant"
)
else:
result.outcome = "unknown"
result.add_warning(
"File not referenced in any platform config - needs manual review"
"Not recognised: no platform list or emulator profile declares this "
"file. Say in the pull request which emulator reads it, for which "
"system, and where the dump comes from"
)
normalized = os.path.normpath(relative)
@@ -452,11 +480,27 @@ def validate_file(
result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/")
else:
result.add_warning("File should be in bios/Manufacturer/Console/ structure")
elif result.outcome == "held":
pass
elif expected_folders:
# The contributor asks before forking: say where it goes.
names = {
name
for table in (platform_hashes, profile_hashes)
for value in (result.sha1, result.md5, hashes.get("sha256", ""))
for name in table.get("named", {}).get(value, ())
}
name = next(iter(names)) if len(names) == 1 else ""
if len(expected_folders) == 1 and name and "/" not in name:
folder = expected_folders[0]
result.destination = (
f"{folder}.variants/{name}.{result.sha1[:8]}"
if os.path.exists(folder + name) else folder + name
)
result.add_warning(
"File is not under bios/ yet. Its system's files are in "
+ " or ".join(f"`{f}`" for f in expected_folders)
+ (f". --place copies it to `{result.destination}`" if result.destination else "")
)
elif _variant_home(db, result.filename, result.sha1):
result.add_warning(
@@ -485,9 +529,11 @@ def validate_file(
if (name_known and not (sha1_known or md5_known or other_known or accepted_by)
and not by_profile and size_fits):
home = _variant_home(db, result.filename)
slot = f"{home}.variants/{result.filename}.{result.sha1[:8]}"
if home and not normalized.startswith("bios" + os.sep):
result.destination = slot
result.add_info(
"This may be a valid variant. If accepted, it will be placed in "
f"`{home}.variants/{result.filename}.{result.sha1[:8]}`"
f"This may be a valid variant. If accepted, it will be placed in `{slot}`"
)
return result
@@ -558,30 +604,35 @@ def _repo_key(url: str) -> str:
def _project_main_line() -> str | None:
"""The project's main line, fetched when no remote of this clone names it.
"""The project's main line, fetched when this clone does not hold it.
A fork's clone has origin alone. Once the commit is pushed to the fork's
main, origin/main is HEAD and the diff against it was empty, so --changed
found nothing and exited 0. None leaves the upstream/ and origin/ lines
in charge: a remote of that name names the project, or the clone has no
remote at all and only its uncommitted files can be read.
found nothing and exited 0. A remote added as CONTRIBUTING says and never
fetched did the same: upstream/main did not exist and origin/main won.
None leaves the upstream/ and origin/ lines in charge when the clone has
no remote at all and only its uncommitted files can be read.
"""
remotes = _git_lines("remote") or []
if not remotes:
return None
project = {_repo_key(url) for url in PROJECT_URLS}
source = PROJECT_URLS[0]
for remote in remotes:
urls = _git_lines("remote", "get-url", "--all", remote) or []
if any(_repo_key(url) in project for url in urls):
return None if remote in ("upstream", "origin") else f"{remote}/main"
if _git_lines("rev-parse", "--verify", "--quiet", f"{remote}/main"):
return f"{remote}/main"
source = remote
break
fetched = subprocess.run(
["git", "fetch", "--quiet", "--no-tags", PROJECT_URLS[0], "main"],
["git", "fetch", "--quiet", "--no-tags", source, "main"],
capture_output=True, text=True, check=False,
)
if fetched.returncode != 0:
raise RuntimeError(
f"no remote of this clone is {PROJECT_URLS[0]}, and fetching its "
f"main failed: {fetched.stderr.strip()}"
f"this clone does not hold the main of {PROJECT_URLS[0]}, and "
f"fetching its main from {source} failed: {fetched.stderr.strip()}"
)
return "FETCH_HEAD"
@@ -675,6 +726,28 @@ def _files_to_validate(
return files
def place(results: list[ValidationResult]) -> list[str]:
"""Copy each placeable file to its destination, never over a file.
The report named the folder of each file and left 40 copies from a card
to be made by hand, each .variants/ name retyped. The source is copied,
not moved: it is often a card or another tree the contributor keeps.
"""
placed = []
for r in results:
if not (r.passed and r.destination and not r.outcome):
continue
target = Path(r.destination)
if target.exists():
print(f"not placed: {target} exists", file=sys.stderr)
continue
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(r.filepath, target)
placed.append(r.destination)
print(f"placed {r.filepath} at {r.destination}", file=sys.stderr)
return placed
def _load_database_if_any(path: str) -> dict | None:
"""The database, optional only where the default is meant.
@@ -709,6 +782,11 @@ def main() -> None:
"--markdown", action="store_true", help="Output as markdown (for PR comments)"
)
output.add_argument("--json", action="store_true", help="Output as JSON")
parser.add_argument(
"--place", action="store_true",
help="copy each recognised file from outside bios/ to the folder and name "
"the report gives, or to .variants/ beside the name it varies",
)
args = parser.parse_args()
# The database, platforms and profiles are read from the repository
@@ -745,6 +823,8 @@ def main() -> None:
{
"file": r.filepath,
"passed": r.passed,
"outcome": r.outcome or "new",
"destination": r.destination,
"sha1": r.sha1,
"md5": r.md5,
"size": r.size,
@@ -754,7 +834,15 @@ def main() -> None:
print(json.dumps(output, indent=2))
elif args.markdown:
lines = ["## BIOS Validation Report", ""]
status = "all checks passed" if all_passed else "some checks failed"
held = sum(r.outcome == "held" for r in results)
unknown = sum(r.outcome == "unknown" for r in results)
review = ", ".join(part for part in (
f"{held} already held" if held else "",
f"{unknown} not recognised" if unknown else "",
) if part)
status = ("some checks failed" if not all_passed
else f"no check failed, to review: {review}" if review
else "all checks passed")
lines.append(f"**Status**: {status}")
lines.append("")
@@ -765,8 +853,7 @@ def main() -> None:
print("\n".join(lines))
else:
for r in results:
status = "PASS" if r.passed else "FAIL"
print(f"\n[{status}] {r.filepath}")
print(f"\n[{r.status}] {r.filepath}")
if r.sha1:
print(f" SHA1: {r.sha1}")
print(f" MD5: {r.md5}")
@@ -783,6 +870,9 @@ def main() -> None:
)
print(f" [{marker}] {m}")
if args.place:
place(results)
if not all_passed:
sys.exit(1)