From bd75defb61ac6110c317b879698f5a9e4dcc39c1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:08:18 +0200 Subject: [PATCH] fix: date sitemap entries from their pages The sitemap gave every page the build date. A page now states its own date and a mkdocs hook passes it on, or leaves lastmod out. The values added to the site's code spans today are also escaped: a backtick in a profile or file name could close the span and inject markup. --- .github/workflows/deploy-site.yml | 1 + mkdocs.yml | 3 +++ scripts/generate_site.py | 22 ++++++++++++++------- scripts/mkdocs_hooks.py | 17 ++++++++++++++++ scripts/siterender.py | 10 ++++++++++ tests/test_audit_regressions.py | 32 +++++++++++++++++++++++++++++++ tests/test_workflow_paths.py | 10 ++++++++++ 7 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 scripts/mkdocs_hooks.py diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 01362ca3..368c9590 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -20,6 +20,7 @@ on: - "scripts/generate_site.py" - "scripts/hashing.py" - "scripts/largefiles.py" + - "scripts/mkdocs_hooks.py" - "scripts/nativemode.py" - "scripts/packextras.py" - "scripts/packpaths.py" diff --git a/mkdocs.yml b/mkdocs.yml index d57622d2..ab6ca2e8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,9 @@ repo_name: Abdess/retrobios # Almost every page is generated from platforms/, emulators/ and database.json, # so a per-page edit link would point at a file that does not exist. edit_uri: '' +# lastmod in the sitemap from each page's own date, or none at all. +hooks: + - scripts/mkdocs_hooks.py # Working notes kept under docs/ during development are not part of the site. exclude_docs: | plans/ diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 877770af..2e8951fc 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1309,7 +1309,9 @@ def decorate_markdown_pages(docs: Path) -> None: "generated_by: retrobios-site\n" f"title: {json.dumps(browser_title, ensure_ascii=False)}\n" f"description: {json.dumps(description, ensure_ascii=False)}\n" - "---\n\n" + + (f"date_modified: {json.dumps(str(structured['dateModified']))}\n" + if structured.get("dateModified") else "") + + "---\n\n" '\n\n" @@ -1916,14 +1918,14 @@ def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]: # The card named a file, its hashes and provenance, but not where to # get it: a user missing scph5501.bin found the reference and no way # to the bytes. - lines.append(f"- Collection path: `{f['path']}` ([download]({_fetch_url(f['path'])}))") + lines.append(f"- Collection path: {_code(f['path'])} ([download]({_fetch_url(f['path'])}))") # Every hash in full: the site search indexes text, not a title # attribute, so a pasted md5 or crc32 (what clrmamepro, RomVault or an # emulator log prints) found nothing behind a 12-character prefix. for label, value in (("SHA1", sha1_full), ("MD5", md5_full), ("CRC32", f.get("crc32")), ("SHA256", f.get("sha256"))): if value and value != "unknown": - lines.append(f'- {label}: `{value}`') + lines.append(f'- {label}: {_code(value)}') if plats: plat_badges = " ".join( f'[{p}](../platforms/{p}.md)' @@ -2293,7 +2295,7 @@ def _render_emulator_file( # byte-swapped 16-bit words, so the raw Adler-32 the downloads carry # for the same file differs, and the card says which one it lists. values = fadler32 if isinstance(fadler32, list) else [fadler32] - shown = ", ".join(f"`{str(v).lower().removeprefix('0x')}`" for v in values) + shown = ", ".join(_code(str(v).lower().removeprefix("0x")) for v in values) if f.get("adler32_byteswap"): details.append( f"Adler32 over the file's 16-bit words byte-swapped: {shown} " @@ -2328,7 +2330,8 @@ def _render_emulator_file( if len(parts) >= 3 and parts[0] == "bios": page = parts[1].lower().replace(" ", "-") details.append( - f"Collection copy: [`{held}`](../systems/{page}.md#sha1-{sha1[:12]})" + f"Collection copy: [{_code(held)}]" + f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" ) plats = sorted(consumers.platforms.get(sha1, ())) if sha1 else [] if plats: @@ -2895,7 +2898,8 @@ def _render_problem_files(platform_problems: list, settled: list | None = None) ]) for p in sorted(settled, key=lambda row: (row["name"], row["platform"])): lines.append( - f"| `{p['name']}` | {p['platform']} | {p['status']} | `{p['held_by']}` |" + f"| {_code(p['name'])} | {_escape_tags(p['platform'])} | {p['status']} " + f"| {_code(p['held_by'])} |" ) lines.append("") @@ -3318,7 +3322,7 @@ def generate_gap_analysis( emulators = sorted({emulator for emulator, _name in bundled_rows}) lines.extend([ f"{len(bundled_rows)} further files arrive with their emulator's own " - f"package ({', '.join(emulators)}): documented in the profile, not " + f"package ({_escape_tags(', '.join(emulators))}): documented in the profile, not " "collected, and not listed here.", "", ]) @@ -4178,6 +4182,9 @@ repo_name: Abdess/retrobios # Almost every page is generated from platforms/, emulators/ and database.json, # so a per-page edit link would point at a file that does not exist. edit_uri: '' +# lastmod in the sitemap from each page's own date, or none at all. +hooks: + - scripts/mkdocs_hooks.py # Working notes kept under docs/ during development are not part of the site. exclude_docs: | plans/ @@ -4323,6 +4330,7 @@ from siterender import ( # noqa: E402,F401 _fmt_size, _pct, _admonition_body, + _code, _escape_tags, _icon_name, _icon_url, diff --git a/scripts/mkdocs_hooks.py b/scripts/mkdocs_hooks.py new file mode 100644 index 00000000..9d6bba27 --- /dev/null +++ b/scripts/mkdocs_hooks.py @@ -0,0 +1,17 @@ +"""MkDocs hooks for the generated site. + +The sitemap gave all 664 pages the build date as lastmod, so the field told +a crawler nothing. A page whose content carries its own date (an emulator +page: the day its profile was last read against the source) states it in +its front matter; the rest get no lastmod rather than an inexact one. +""" + +from __future__ import annotations + +from typing import Any + + +def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001 + """Take lastmod from the page's own date, or leave it out.""" + page.update_date = str(page.meta.get("date_modified") or "") + return markdown diff --git a/scripts/siterender.py b/scripts/siterender.py index ee84ce86..6a10fb3f 100644 --- a/scripts/siterender.py +++ b/scripts/siterender.py @@ -20,6 +20,16 @@ _icon_available: dict[str, bool] = {} _CODE_SPAN = re.compile(r"(`+).+?\1", re.S) +def _code(value: object) -> str: + """A value as an inline code span it cannot leave. + + Markdown shows HTML inside a code span as text, so the one way out is a + backtick in the value closing the span early. Data from a profile, a + platform file or a file name is not ours to trust that far. + """ + return "`" + str(value).replace("`", "'") + "`" + + def _escape_tags(text: str) -> str: """Show a placeholder such as ```` as it was written. diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index b62115ef..e36b82d9 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4638,5 +4638,37 @@ class VersionAnchorsFollowContent(unittest.TestCase): ["file-audio-abs-6904b-v1-12", "file-audio-abs-6904b-v1-13"]) + +class SitemapDatesComeFromThePage(unittest.TestCase): + """All 664 sitemap URLs carried the build date: eduke32, re-read on + 2026-10-04, was announced as changed on the day of every build.""" + + def test_the_hook_takes_the_page_date_or_none(self): + from types import SimpleNamespace + + from scripts.mkdocs_hooks import on_page_markdown + + dated = SimpleNamespace(meta={"date_modified": "2026-10-04"}, update_date="today") + undated = SimpleNamespace(meta={}, update_date="today") + on_page_markdown("", dated, None, None) + on_page_markdown("", undated, None, None) + self.assertEqual((dated.update_date, undated.update_date), ("2026-10-04", "")) + config = (ROOT / "mkdocs.yml").read_text(encoding="utf-8") + self.assertIn("hooks:\n - scripts/mkdocs_hooks.py", config) + + +class SiteCodeSpansHoldTheirValue(unittest.TestCase): + """Values from profiles, platform files and file names went into code + spans as they were: a backtick in one closed the span and what followed + was read as markup.""" + + def test_a_backtick_cannot_close_the_span(self): + from scripts.siterender import _code + + span = _code("a``") + self.assertEqual(span.count("`"), 2) + self.assertTrue(span.startswith("`") and span.endswith("`")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py index 64e4de62..3c66765b 100644 --- a/tests/test_workflow_paths.py +++ b/tests/test_workflow_paths.py @@ -9,6 +9,7 @@ from __future__ import annotations import ast import fnmatch +import re import sys import unittest from pathlib import Path @@ -78,6 +79,15 @@ class DeploySiteTriggers(unittest.TestCase): } ) self.assertIn("generate_site.py", entry_points) + # mkdocs build runs the hooks mkdocs.yml names: a script of the build. + # The file carries Python tags safe_load refuses; the list is plain. + mkdocs = (REPO_ROOT / "mkdocs.yml").read_text(encoding="utf-8") + hooks = re.search(r"^hooks:\n((?: - .+\n)+)", mkdocs, re.M) + entry_points += [ + line.strip()[2:].removeprefix("scripts/") + for line in (hooks.group(1).splitlines() if hooks else []) + if line.strip()[2:].startswith("scripts/") + ] uncovered = sorted( path for path in closure(entry_points)