chore: merge origin/main

scripts/fetch_site_images.py

@@ -52,14 +52,33 @@ def fetch(url: str) -> bytes:
         return response.read()
 
 
-def square_image(data: bytes, size: int, kind: str = "PNG") -> bytes:
+def square_image(data: bytes, size: int, kind: str = "PNG", flip: bool = False,
+                 trim: bool = False) -> bytes:
+    """*flip* mirrors the image top to bottom, for a boot logo stored in the
+    row order a device's screen reads (MinUI's). *trim* cuts the plain
+    background around the mark, the corner pixel's colour, and keeps that
+    colour around it: a 640 x 480 boot screen became a dot."""
     try:
-        from PIL import Image
+        from PIL import Image, ImageChops
     except ImportError as exc:
         raise SystemExit("Pillow is needed to normalise raster images: pip install Pillow") from exc
     image = Image.open(io.BytesIO(data)).convert("RGBA")
+    if flip:
+        image = image.transpose(Image.Transpose.FLIP_TOP_BOTTOM)
+    background = (0, 0, 0, 0)
+    if trim:
+        background = image.getpixel((0, 0))
+        # Faint noise in the background is no part of the mark.
+        box = (ImageChops.difference(image, Image.new("RGBA", image.size, background))
+               .convert("L").point(lambda value: 255 if value > 48 else 0).getbbox())
+        if box:
+            image = image.crop(box)
+        side = round(max(image.size) * 1.2)
+        framed = Image.new("RGBA", (side, side), background)
+        framed.paste(image, ((side - image.width) // 2, (side - image.height) // 2))
+        image = framed
     image.thumbnail((size, size), Image.LANCZOS)
-    tile = Image.new("RGBA", (size, size), (0, 0, 0, 0))
+    tile = Image.new("RGBA", (size, size), background)
     tile.paste(image, ((size - image.width) // 2, (size - image.height) // 2), image)
     out = io.BytesIO()
     if kind == "WEBP":
@@ -130,14 +149,19 @@ def write_manifest(directory: Path, note: str, entries: list[dict]) -> None:
 
 
 def fetch_logos() -> None:
-    from siteguides import GUIDES
+    from siteguides import GROUP_TILES, GUIDES
 
     with open(ROOT / "platforms" / "_registry.yml", encoding="utf-8") as handle:
         platforms = yaml.safe_load(handle)["platforms"]
     LOGO_DIR.mkdir(parents=True, exist_ok=True)
     entries = []
-    for name, entry in sorted(platforms.items()):
-        url = entry.get("logo")
+    # The tiles of a kind of device (Android handhelds) beside the setups
+    # carry a logo too, named after the tile.
+    sources = [(name, entry.get("logo"), entry, GUIDES.get(name))
+               for name, entry in sorted(platforms.items())]
+    sources += [(re.sub(r"[^a-z0-9]+", "-", tile.name.lower()).strip("-"), tile.logo, {}, tile)
+                for tiles in GROUP_TILES.values() for tile in tiles]
+    for name, url, entry, guide in sources:
         if not url:
             # No logo of its own: the tile shows initials, and an old copy
             # would still be served.
@@ -149,13 +173,14 @@ def fetch_logos() -> None:
         if is_svg(data):
             file_name, payload = f"{name}.svg", minify_svg(shrink_embedded(safe_svg(data, url)))
         else:
-            file_name, payload = f"{name}.webp", square_image(data, LOGO_SIZE, "WEBP")
+            file_name, payload = f"{name}.webp", square_image(
+                data, LOGO_SIZE, "WEBP", flip=bool(entry.get("logo_flip")),
+                trim=bool(entry.get("logo_trim")))
         for stale in LOGO_DIR.glob(f"{name}.*"):
             if stale.name != file_name:
                 stale.unlink()
         (LOGO_DIR / file_name).write_bytes(payload)
         # Who owns each mark: logo_owner and logo_licence in the guides.
-        guide = GUIDES.get(name)
         owner = (guide and guide.logo_owner) or name
         licence = (guide and guide.logo_licence) or "ownership not recorded"
         entries.append({

scripts/generate_installer_doc.py

@@ -586,7 +586,8 @@ def sample_writers(box: Sandbox) -> dict[str, Callable[[str], str]]:
             f"{list_date('retroarch')}:\n\n"
             + _fenced([_bulk_line(box)])),
         "run-check": lambda _body: _fenced(
-            _lines_from(check_run(), "Checking existing files", "  Everything is in place")),
+            _lines_from(check_run(), "Checking existing files",
+                        "  Every file on the list is in place")),
         "check-refused": lambda _body: _fenced(_refused_lines(box.small("refused"))),
         "from-run": lambda _body: _fenced(
             _lines_from(box.small("from"), "Done.", count=2)),

scripts/generate_site.py

@@ -442,7 +442,8 @@ def _tiles(registry: dict, members: list[str], display: dict[str, str], prefix:
            str(registry.get(name, {}).get("logo") or ""),
            GUIDES[name].runs_on or _runs_on(registry.get(name, {}), GUIDES[name]))
           for name in members),
-        *((tile.name, f"{prefix}{tile.link}", "", tile.text) for tile in GROUP_TILES.get(group, ())),
+        *((tile.name, f"{prefix}{tile.link}", tile.logo, tile.text)
+          for tile in GROUP_TILES.get(group, ())),
     ], eager=eager)
 
 
@@ -507,7 +508,7 @@ def generate_home(
         "## Is it needed? { #do-i-need-it }",
         "",
         "- **Yes**, if the emulator says a BIOS is missing.",
-        "- **Yes** for PlayStation 1 and 2, Saturn and Neo Geo.",
+        "- **Yes** for PlayStation 2 and Neo Geo.",
         "- **Better with it** for Dreamcast: some games need it.",
         "- **Usually not** for NES, Super Nintendo, Mega Drive and Game Boy.",
         "",
@@ -4202,7 +4203,7 @@ def check_pass_line() -> str:
     no file is missing or wrong (install.py EVERYTHING_IN_PLACE), read from
     the installer. Typed here, it kept "All files up to date." after the
     installer said "Everything is in place."."""
-    return f"All good when it says `{_installer().EVERYTHING_IN_PLACE}`"
+    return f"Good result: `{_installer().EVERYTHING_IN_PLACE}`"
 
 # Under "From your computer": a device with no network installs from the ZIP
 # (`--from-pack`). The id is the one wiki/installer.md gives its section.

scripts/indexnow.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""The pages a deploy changes, as the IndexNow list a search engine takes.
+
+A search engine revisits a page on its own schedule, and an assistant that
+searches through it reads the copy it last saw. IndexNow (Bing, Yandex,
+Seznam, Naver) takes a list of changed addresses and fetches them soon
+after. Google does not take it and reads the sitemap.
+
+The built site carries `digests.json`, each page's SHA-256 before its build
+line (scripts/site_markdown.py). This compares it with the one the live site
+serves, before the deploy replaces it: a page whose digest differs, a new
+page and a removed one are listed. The deploy job posts the list once the
+new site is live (.github/workflows/deploy-site.yml).
+
+    python scripts/indexnow.py --dist website/dist              # print the list
+    python scripts/indexnow.py --dist website/dist --output "$GITHUB_OUTPUT"
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(__file__))
+from siteagents import INDEXNOW_FILE, INDEXNOW_KEY, SITE  # noqa: E402
+
+HOST = "abdess.github.io"
+# What one IndexNow request accepts.
+MAX_URLS = 10_000
+USER_AGENT = "retrobios-indexnow"
+
+
+def live_digests(url: str = f"{SITE}digests.json") -> dict[str, str]:
+    """The digests the live site serves. None yet (the first deploy that
+    writes them) is an empty map: every page is new."""
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(request, timeout=60) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        if exc.code == 404:
+            return {}
+        raise
+
+
+def changed_urls(new: dict[str, str], old: dict[str, str]) -> list[str]:
+    """Pages added, changed or removed between two digest maps."""
+    return sorted(url for url in new.keys() | old.keys() if new.get(url) != old.get(url))
+
+
+def payload(urls: list[str]) -> dict:
+    if len(urls) > MAX_URLS:
+        raise SystemExit(f"error: {len(urls)} changed pages, one request takes {MAX_URLS}")
+    return {"host": HOST, "key": INDEXNOW_KEY, "keyLocation": f"{SITE}{INDEXNOW_FILE}",
+            "urlList": urls}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False)
+    parser.add_argument("--dist", type=Path, required=True, help="the built site (website/dist)")
+    parser.add_argument("--output", type=Path,
+                        help="append payload=<json> to this file (a GitHub step output)")
+    args = parser.parse_args()
+    new = json.loads((args.dist / "digests.json").read_text(encoding="utf-8"))
+    urls = changed_urls(new, live_digests())
+    print(f"{len(urls)} of {len(new)} pages changed", file=sys.stderr)
+    if not urls:
+        return 0
+    body = json.dumps(payload(urls), separators=(",", ":"))
+    if args.output:
+        with args.output.open("a", encoding="utf-8") as handle:
+            handle.write(f"payload={body}\n")
+    else:
+        print(body)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/site_markdown.py

@@ -11,6 +11,13 @@ website/astro.config.mjs), and llms.txt says the convention.
 `llms-full.txt` joins the Markdown of the pages a setup is done with:
 the start pages, the guides and each setup's page.
 
+An assistant often reads a copy fetched long before, by itself or by a
+search engine. Each copy, llms.txt and llms-full.txt name the commit and
+the date they were built from, and `version.json` names the current one,
+so a stale copy can be told apart. `digests.json` gives each page's
+SHA-256 before that line: the next deploy compares it with its own and
+tells search engines which pages changed (scripts/indexnow.py).
+
 Run by the Astro build once the pages are written (astro.config.mjs),
 on the built site:
 
@@ -20,8 +27,11 @@ on the built site:
 from __future__ import annotations
 
 import argparse
+import hashlib
+import json
 import os
 import re
+import subprocess
 import sys
 from html.parser import HTMLParser
 from pathlib import Path
@@ -293,42 +303,86 @@ def page_markdown(html: str, page_url: str) -> str:
     return converter.markdown()
 
 
-def convert_site(dist: Path) -> int:
-    """Write index.md beside every page's index.html, then llms-full.txt."""
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def build_stamp() -> dict[str, str]:
+    """The commit the site is built from and its date. CI names the commit
+    in GITHUB_SHA. The date is the commit's, so two builds of one commit
+    write the same bytes."""
+    commit = os.environ.get("GITHUB_SHA") or subprocess.run(
+        ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True,
+        check=True).stdout.strip()
+    date = subprocess.run(["git", "log", "-1", "--format=%cI", commit], cwd=ROOT,
+                          capture_output=True, text=True, check=True).stdout.strip()
+    return {"commit": commit, "date": date}
+
+
+def stamp_line(stamp: dict[str, str]) -> str:
+    return (f"> Built from commit {stamp['commit'][:12]} on {stamp['date'][:10]}. "
+            f"The current build: {SITE}{BASE}version.json")
+
+
+def stamped(text: str, stamp: dict[str, str]) -> str:
+    """*text* with the build line under its first line, the title."""
+    first, _, rest = text.partition("\n")
+    return f"{first}\n\n{stamp_line(stamp)}\n{rest}"
+
+
+def convert_site(dist: Path, stamp: dict[str, str] | None = None) -> int:
+    """Write index.md beside every page's index.html, then llms-full.txt,
+    version.json and digests.json, and stamp llms.txt."""
+    stamp = stamp or build_stamp()
     written = 0
+    digests: dict[str, str] = {}
     for html_path in sorted(dist.rglob("index.html")):
         relative = html_path.parent.relative_to(dist).as_posix()
         page = "" if relative == "." else f"{relative}/"
         url = f"{SITE}{BASE}{page}"
         text = page_markdown(html_path.read_text(encoding="utf-8"), url)
-        (html_path.parent / "index.md").write_text(text, encoding="utf-8")
+        digests[url] = hashlib.sha256(text.encode("utf-8")).hexdigest()
+        (html_path.parent / "index.md").write_text(stamped(text, stamp), encoding="utf-8")
         written += 1
-    write_full(dist)
+    write_full(dist, stamp)
+    llms = dist / "llms.txt"
+    if llms.is_file():
+        text = llms.read_text(encoding="utf-8")
+        digests[f"{SITE}{BASE}llms.txt"] = hashlib.sha256(text.encode("utf-8")).hexdigest()
+        # Under the summary, which llms.txt keeps as the quote after its title.
+        title, summary, rest = text.split("\n\n", 2)
+        line = stamp_line(stamp).removeprefix("> ")
+        llms.write_text(f"{title}\n\n{summary}\n\n{line}\n\n{rest}", encoding="utf-8")
+    (dist / "version.json").write_text(json.dumps(stamp, indent=1) + "\n", encoding="utf-8")
+    (dist / "digests.json").write_text(
+        json.dumps(dict(sorted(digests.items())), indent=0) + "\n", encoding="utf-8")
     return written
 
 
-def write_full(dist: Path) -> None:
+def _unstamped(text: str, stamp: dict[str, str]) -> str:
+    """A page's copy without its build line: llms-full.txt says it once."""
+    return text.replace(f"\n\n{stamp_line(stamp)}\n", "\n", 1)
+
+
+def write_full(dist: Path, stamp: dict[str, str]) -> None:
     """llms-full.txt from the pages' Markdown copies."""
-    parts = ["# RetroBIOS: the guides in one file", "",
+    parts = ["# RetroBIOS: the guides in one file", "", stamp_line(stamp), "",
              "> The pages a setup is done with, in reading order. Every page of the site also has "
              "its own Markdown copy at `<page>/index.md`.", ""]
     for page in FULL_PAGES:
         md = dist / page / "index.md"
         if md.is_file():
-            parts += [md.read_text(encoding="utf-8").rstrip(), "", "---", ""]
+            parts += [_unstamped(md.read_text(encoding="utf-8"), stamp).rstrip(), "", "---", ""]
     # Each setup's own page (the setups the picker lists, not the parts a
     # long page is cut into), up to its reference of every file: what to
     # do and where, not the full lists, which its own copy keeps. All of
     # them was 6.9 MB, past what an assistant reads at once.
-    import json  # noqa: PLC0415
-
     index = dist / "data" / "picker" / "index.json"
     setups = [str(item["platform"]) for item in json.loads(index.read_text(encoding="utf-8"))] \
         if index.is_file() else []
     for setup in [*setups, "linux-handhelds"]:
         md = dist / "platforms" / setup / "index.md"
         if md.is_file():
-            text = md.read_text(encoding="utf-8")
+            text = _unstamped(md.read_text(encoding="utf-8"), stamp)
             parts += [text.split("\n## Reference", 1)[0].rstrip(), "", "---", ""]
     (dist / "llms-full.txt").write_text("\n".join(parts).rstrip() + "\n", encoding="utf-8")
 

scripts/siteagents.py

@@ -73,8 +73,8 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
         "code: which files an emulator reads, whether it starts without them, where each one "
         "goes on every setup, their hashes, and an installer that puts them in place.",
         "",
-        "Every claim about a file cites the line of emulator source code that reads it. The "
-        "files themselves are served by the GitHub repository "
+        "Each emulator's files are read from its code, and most cite the file and line that "
+        "read them. The files themselves are served by the GitHub repository "
         "(https://github.com/Abdess/retrobios) and its releases.",
         "",
         "## Answering a user",
@@ -131,6 +131,9 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
 # folders and hashes change with each setup, and a guessed one is the
 # answer that fails. The same rules end the skill.
 ANSWER_RULES = (
+    "Each Markdown copy, llms.txt and llms-full.txt name the commit and the date they "
+    f"were built from. When a copy read earlier or found through a search engine names a "
+    f"commit other than {SITE}version.json, fetch it again: the site changes with each commit.",
     "Answer from the pages and data, not from memory. Every page has a Markdown copy: add "
     "`index.md` to its address (https://abdess.github.io/retrobios/which-pack/index.md). "
     f"[llms-full.txt]({SITE}llms-full.txt) holds the guides and every setup's page in one file.",
@@ -247,9 +250,16 @@ def ard_manifest() -> dict:
     }
 
 
+# IndexNow: the key search engines read back from the site before they take
+# a list of changed pages from it (scripts/indexnow.py). Public by design.
+INDEXNOW_KEY = "23329654933fb67280e8e7ac749eb869"
+INDEXNOW_FILE = "indexnow.txt"
+
+
 def write(public: Path, platforms: list[tuple[str, str]],
           write_if_changed: Callable[[str, str], object]) -> None:
     write_if_changed(str(public / "llms.txt"), llms_txt(platforms))
+    write_if_changed(str(public / INDEXNOW_FILE), INDEXNOW_KEY + "\n")
     (public / SKILL_PATH).parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(str(public / SKILL_PATH), skill_md())
     write_if_changed(str(public / "ard.json"), json.dumps(ard_manifest(), indent=1) + "\n")

scripts/siteguides.py

@@ -81,6 +81,10 @@ class Tile:
     name: str
     text: str
     link: str
+    # A logo naming the kind of device, credited like a setup's.
+    logo: str = ""
+    logo_owner: str = ""
+    logo_licence: str = ""
 
 
 @dataclass(frozen=True)
@@ -129,6 +133,7 @@ _TOP_KEYS = {"install_base", "trust_items", "device_groups", "terminals", "guide
 _GROUP_KEYS = {"label", "description", "anchor", "also_runs", "tiles"}
 _ALSO_KEYS = ("setup", "summary", "uses", "runs_on", "folder")
 _TILE_KEYS = ("name", "text", "link")
+_TILE_LOGO_KEYS = ("logo", "logo_owner", "logo_licence")
 _HANDHELD_KEYS = {"page", "title", "summary", "uses", "systems"}
 _SYSTEM_KEYS = {"name", "folder", "folder_source", "devices", "devices_source", "linux_only"}
 _SOURCE = re.compile(r"https://\S+")
@@ -342,8 +347,12 @@ def load_guides(path: Path = GUIDES_FILE, registry: dict | None = None) -> dict:
             row_at = f"{at}.tiles[{i}]"
             if not isinstance(row, dict):
                 raise GuideError(f"{row_at}: expected a mapping")
-            _unknown(row, _TILE_KEYS, row_at)
-            group_tiles.append(Tile(*(_text(row.get(k), f"{row_at}.{k}") for k in _TILE_KEYS)))
+            _unknown(row, _TILE_KEYS + _TILE_LOGO_KEYS, row_at)
+            if row.get("logo") and not all(row.get(k) for k in _TILE_LOGO_KEYS):
+                raise GuideError(f"{row_at}: a logo needs logo_owner and logo_licence")
+            group_tiles.append(Tile(
+                *(_text(row.get(k), f"{row_at}.{k}") for k in _TILE_KEYS),
+                *(_text(row[k], f"{row_at}.{k}") if row.get(k) else "" for k in _TILE_LOGO_KEYS)))
         if group_tiles:
             tiles[key] = tuple(group_tiles)