chore: merge main into the p3 fixes

scripts/generate_event_schema.py

@@ -45,7 +45,7 @@ KEY_TYPES: dict[str, dict] = {
         "pending", "code",
     ), _COUNT),
     **dict.fromkeys(("truncated", "ok", "interrupted"), _FLAG),
-    **dict.fromkeys(("systems", "other_drives", "dests"), _NAMES),
+    **dict.fromkeys(("systems", "other_drives", "dests", "written"), _NAMES),
     **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept"), _RECORDS),
 }
 

scripts/generate_launcher_index.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Write install/launchers/index.json: the handheld launchers as files for a card.
+
+    python scripts/generate_launcher_index.py           rewrite the list
+    python scripts/generate_launcher_index.py --check   exit 1 when it is stale
+
+install.py --add-app reads this list from the ref its file lists come from,
+and puts on a handheld's card the launcher of its system, each file checked
+against the size and SHA-256 recorded here. The files are the members of the
+archives scripts/build_port.py builds, from the same sources: muOS gets its
+application archive, rebuilt by the installer from the members and checked
+against the hash recorded for it, the others the archive's files at their
+place on the card. Where each system's files go is install.py's APP_PLACES.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+from build_port import (
+    BATOCERA_ZIP,
+    MUOS_APP,
+    REPO_ROOT,
+    TOOLS_ZIP,
+    _packs_table,
+    archive_members,
+    pinned_installer_sha256,
+)
+
+OUTPUT = REPO_ROOT / "install" / "launchers" / "index.json"
+
+_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py")
+_install = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_install)
+
+# The archive whose members each system gets, by its key in APP_PLACES.
+LAUNCHER_ARCHIVES = {"muos": MUOS_APP, "knulli": BATOCERA_ZIP, "rocknix": TOOLS_ZIP}
+
+
+def _entry(field: str, name: str, source: Path | None, root: Path) -> dict:
+    """A member as the list records it: where it comes from, its size and
+    hash, and whether it is executable in the repository."""
+    if source is None:
+        data = _packs_table(root)
+        origin: dict = {"content": data.decode("utf-8")}
+        executable = False
+    else:
+        data = source.read_bytes()
+        origin = {"repo_path": source.relative_to(root).as_posix()}
+        executable = bool(source.stat().st_mode & 0o111)
+    return {field: name, **origin, "size": len(data),
+            "sha256": hashlib.sha256(data).hexdigest(), "executable": executable}
+
+
+def _member_bytes(entry: dict, root: Path) -> bytes:
+    if "content" in entry:
+        return entry["content"].encode("utf-8")
+    return (root / entry["repo_path"]).read_bytes()
+
+
+def build_index(root: Path = REPO_ROOT) -> dict:
+    """Every launcher a card can carry, from the archives' members."""
+    actual = hashlib.sha256((root / "install.py").read_bytes()).hexdigest()
+    if actual != pinned_installer_sha256(root):
+        raise ValueError("install.py is not the installer install.sh pins: re-pin the "
+                         "bootstraps before writing the launcher list")
+    places = {place.key: place for place in _install.APP_PLACES.values()}
+    if set(places) != set(LAUNCHER_ARCHIVES):
+        raise ValueError(f"APP_PLACES names {sorted(places)}, the archives "
+                         f"{sorted(LAUNCHER_ARCHIVES)}")
+    launchers: dict[str, dict] = {}
+    for key, archive in sorted(LAUNCHER_ARCHIVES.items()):
+        folder = places[key].folder
+        members = archive_members(archive, root)
+        if archive == MUOS_APP:
+            entries = [_entry("name", name, source, root) for name, source in members]
+            data = _install.app_archive([(entry, _member_bytes(entry, root))
+                                         for entry in entries])
+            launchers[key] = {
+                "archive": {"dest": f"{folder}/{archive}", "size": len(data),
+                            "sha256": hashlib.sha256(data).hexdigest()},
+                "members": entries,
+            }
+        else:
+            # Executable only where the menu starts the script as a program:
+            # a mode a FAT card cannot hold is refused by its driver.
+            files = [_entry("dest", f"{folder}/{name}", source, root)
+                     for name, source in members]
+            for entry in files:
+                entry["executable"] = entry["executable"] and places[key].executable
+            launchers[key] = {"files": files}
+    return {"schema": _install.APP_SCHEMA, "launchers": launchers}
+
+
+def render(index: dict) -> str:
+    return json.dumps(index, indent=2, ensure_ascii=False) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--check", action="store_true",
+                        help="exit 1 when the list differs from the launchers, writing nothing")
+    parser.add_argument("--output", type=Path, default=OUTPUT, help=argparse.SUPPRESS)
+    args = parser.parse_args(argv)
+    try:
+        text = render(build_index())
+    except (OSError, ValueError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1
+    current = args.output.read_text(encoding="utf-8") if args.output.is_file() else None
+    if current == text:
+        return 0
+    if args.check:
+        print(f"{args.output} differs from the launchers in ports/: run "
+              "python scripts/generate_launcher_index.py", file=sys.stderr)
+        return 1
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(text, encoding="utf-8")
+    print(f"Wrote {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

scripts/generate_site.py

@@ -3007,6 +3007,26 @@ def _front_matter(markdown: str) -> dict:
     return data if isinstance(data, dict) else {}
 
 
+_SHA1_BY_PREFIX: dict[str, str] = {}
+
+
+def _cards_elsewhere(markdown: str) -> str:
+    """A start line links each file to its card. A console split over part
+    pages keeps its cards on the parts, so a card this page does not hold
+    is reached by its permanent link instead."""
+    links = set(re.findall(r"\]\(#(sha1-[0-9a-f]{12})\)", markdown))
+    away = {ident for ident in links if f'id="{ident}"' not in markdown}
+    if not away:
+        return markdown
+    if not _SHA1_BY_PREFIX:
+        db = load_database(str(_REPO_ROOT / "database.json"))
+        _SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]})
+    for ident in away:
+        full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")]
+        markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})")
+    return markdown
+
+
 def decorate_markdown_pages(docs: Path) -> dict[str, str]:
     """Add per-page descriptions and machine-readable structured data.
 
@@ -3026,7 +3046,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
             # Written in its final form by an earlier pass and rewritten by
             # nothing since: a second pass would find no title to read.
             continue
-        markdown = prepare_page(original, relative)
+        markdown = _cards_elsewhere(prepare_page(original, relative))
         for image_id in unresolved_images(markdown):
             print(f"  WARNING: {relative}: no diagram named {image_id!r}", file=sys.stderr)
         stripped = _undecorated(markdown)
@@ -5477,8 +5497,14 @@ def _system_file_card(
     # emulator, platform or pack line and looked unfinished.
     unread = (f' <span class="rb-badge rb-badge-muted" {skip}>No reader</span>'
               if not (emus or plats or installed) else "")
+    # The download leads the card: as a "(download)" after the collection
+    # path, fourth of the facts, a reader of the Apple II page found none.
+    download = (
+        f' <a class="rb-download" href="{html.escape(_fetch_url(f["path"]))}" '
+        f'aria-label="Download {html.escape(name)}" {skip}>Download</a>'
+    ) if f.get("path") and markup.starlight() else ""
     lines.append(_card_title(
-        f'{_code(name)} <span class="rb-badge rb-badge-muted">{size}</span>{unread}{permalink}',
+        f'{_code(name)} <span class="rb-badge rb-badge-muted">{size}</span>{unread}{download}{permalink}',
         anchor, searchable,
     ))
     lines.append("")
@@ -5509,8 +5535,10 @@ def _system_file_card(
         # The card named a file, its hashes and provenance, but not where to
         # get it: a user missing scph5501.bin found the reference and no way
         # to the bytes.
-        facts.append(("Collection path",
-                      [f"{_code(f['path'])} ([download]({_fetch_url(f['path'])}))"], False))
+        path = _code(f["path"])
+        if not markup.starlight():
+            path += f" ([download]({_fetch_url(f['path'])}))"
+        facts.append(("Collection path", [path], False))
     catalogued = [line.split(": ", 1)[1]
                   for line in _provenance_lines(f.get("provenance") or {}, "../")]
     if catalogued:
@@ -6196,7 +6224,18 @@ def _start(
             sha1 = sha1s.get(visible)
             if sha1 and sha1 not in order:
                 order.append(sha1)
-    return lines, order
+    return [_link_held_names(line, sha1s, held) for line in lines], order
+
+
+def _link_held_names(line: str, sha1s: dict[str, str], held: set[str]) -> str:
+    """Each file the start line names leads to its card, where it downloads:
+    the Apple II line named eight files and none was a way to get one."""
+    def link(match: re.Match) -> str:
+        sha1 = sha1s.get(match[1], "")
+        return f"[{match[0]}](#sha1-{sha1[:12]})" if sha1 in held else match[0]
+    # A name inside a link already (a "Keep it zipped" archive) stays as
+    # is, and so does the name a file is read under, "(as `d2fdc.zip`)".
+    return re.sub(r"(?<!\[)(?<!\(as )`([^`]*)`(?!\]\()", link, line)
 
 
 def _group_lines(

scripts/profile_sync.py

@@ -4275,6 +4275,7 @@ def _apply_writes(
                     file=sys.stderr,
                 )
                 return
+            _WRITTEN.add(name)
         for message in messages:
             print(message)
 
@@ -4663,16 +4664,20 @@ def main() -> None:
             if not profile_path.is_file():
                 continue
             try:
+                before = profile_path.read_bytes()
                 for line in realign_prose(
                     profile_path, args.cache_dir, args.offline, args.dry_run
                 ):
                     print(f"{name}: {line}")
+                if profile_path.read_bytes() != before:
+                    _WRITTEN.add(name)
             except upstream.UpstreamError as exc:
                 print(f"{name}: {exc}", file=sys.stderr)
                 failed.add(name)
             except YamlWriteError as exc:
                 print(f"{name}: write refused: {exc}", file=sys.stderr)
                 failed.add(name)
+        _refresh_source_paths(args)
         if failed:
             raise SystemExit(1)
         return
@@ -4710,12 +4715,46 @@ def main() -> None:
         if not args.changed_only or report.needs_review() or report.name in failed
     ]
     _emit(args, selected, reports, shown)
+    _refresh_source_paths(args)
     if failed or (args.strict and any(
         report.needs_review() or report.uncheckable() for report in reports
     )):
         raise SystemExit(1)
 
 
+# Profiles this run wrote. Their new pins name trees the site's record of
+# cited source paths does not hold yet.
+_WRITTEN: set[str] = set()
+
+
+def _refresh_source_paths(args: argparse.Namespace) -> None:
+    """Rebuild the site's record of cited source paths after a write.
+
+    Each bump left website/src/data/source-paths.json behind its pins, and
+    the site's freshness test stayed red until someone ran sitesources.py
+    --fetch by hand. A run that writes nothing, or makes no request, leaves
+    it alone.
+    """
+    if not _WRITTEN or args.dry_run or args.offline or os.environ.get("RETROBIOS_OFFLINE"):
+        return
+    import sitesources  # noqa: PLC0415
+    from common import write_if_changed  # noqa: PLC0415
+
+    profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
+    held, unread = sitesources.build_source_paths(
+        profiles, [sitesources.TREE_CACHE], fetch=True
+    )
+    sitesources.SOURCE_PATHS.parent.mkdir(parents=True, exist_ok=True)
+    write_if_changed(
+        str(sitesources.SOURCE_PATHS), json.dumps(held, indent=1, sort_keys=True) + "\n"
+    )
+    print(
+        f"source paths: {sum(map(len, held.values()))} in {len(held)} trees, "
+        f"{len(unread)} trees not cached: {sitesources.SOURCE_PATHS}",
+        file=sys.stderr,
+    )
+
+
 def _build_full_report(
     args: argparse.Namespace,
     name: str,

scripts/sitediagrams.py

@@ -877,7 +877,7 @@ def options_table(markdown: str) -> list[str]:
 GLANCE_HEADING = "## Options at a glance"
 GLANCE_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
     ("Where and for what", ("--platform", "--dest", "--target", "--every-board", "--save-dir",
-                            "--romm-firmware", "--standalone-copies")),
+                            "--romm-firmware", "--standalone-copies", "--add-app")),
     ("Fewer files", ("--system", "--core", "--emulator", "--region", "--no-game-data",
                      "--declared-only", "--installed-cores", "--cores-dir")),
     ("Look first", ("--check", "--dry-run", "--list-platforms", "--list-targets",