From 8022ef4cb31c9140d195863c7e6299773f73e04b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:12:43 +0100 Subject: [PATCH] fix: read the given database on every page The page-finishing pass reloaded the repository's database.json whatever --db said, so a card held only by the given collection raised KeyError. siteinputs holds the inputs generate_site was given. --- scripts/generate_site.py | 25 +++++++++++----- scripts/siteinputs.py | 32 +++++++++++++++++++++ tests/test_site_inputs.py | 60 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 scripts/siteinputs.py create mode 100644 tests/test_site_inputs.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 32f926da0..9b83ff328 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -89,6 +89,7 @@ from generate_readme import ( from provenance_report import build_report, distinct_missing import sitemarkup as markup import siteentries +import siteinputs import retired_files import sitestarlight from siteexports import generate_exports_page, load_export_reports @@ -3174,7 +3175,9 @@ def _front_matter(markdown: str) -> dict: return data if isinstance(data, dict) else {} -_SHA1_BY_PREFIX: dict[str, str] = {} +@functools.lru_cache(maxsize=None) +def _sha1_by_prefix(db_path: str) -> dict[str, str]: + return {sha1[:12]: sha1 for sha1 in load_database(db_path)["files"]} def _cards_elsewhere(markdown: str) -> str: @@ -3185,11 +3188,9 @@ def _cards_elsewhere(markdown: str) -> str: away = {ident for ident in links if f'id="{ident}"' not in markdown} if not away: return markdown - if not _SHA1_BY_PREFIX: - db = load_database(str(_REPO_ROOT / "database.json")) - _SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]}) + by_prefix = _sha1_by_prefix(str(siteinputs.current().db)) for ident in away: - full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")] + full = by_prefix[ident.removeprefix("sha1-")] markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})") return markdown @@ -3500,10 +3501,14 @@ def _list_pages( f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1)) -@functools.lru_cache(maxsize=1) def _collection_sha1_by_path() -> dict[str, str]: """The sha1 of each file the collection holds, by its path.""" - with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle: + return _sha1_by_path_in(str(siteinputs.current().db)) + + +@functools.lru_cache(maxsize=None) +def _sha1_by_path_in(db_path: str) -> dict[str, str]: + with open(db_path, encoding="utf-8") as handle: files = json.load(handle).get("files", {}) return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()} @@ -12260,6 +12265,12 @@ def main() -> None: def _generate(args: argparse.Namespace) -> None: + # Every page reads these, never the repository's own files. + siteinputs.use(siteinputs.SiteInputs( + db=Path(args.db).resolve(), + emulators=Path(args.emulators_dir).resolve(), + platforms=Path(args.platforms_dir).resolve(), + )) db = load_database(args.db) docs = Path(args.docs_dir) # Files the site serves as they are (data, downloads, images), in the diff --git a/scripts/siteinputs.py b/scripts/siteinputs.py new file mode 100644 index 000000000..7aa3326ff --- /dev/null +++ b/scripts/siteinputs.py @@ -0,0 +1,32 @@ +"""The collection, profiles and platform files the site describes. + +generate_site.py takes them as --db, --emulators-dir and --platforms-dir and +installs them here before writing a page. Every page module reads its inputs +from current(), so a run on another collection describes that collection on +every page. A module used on its own reads the repository's files. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +@dataclass(frozen=True) +class SiteInputs: + db: Path = ROOT / "database.json" + emulators: Path = ROOT / "emulators" + platforms: Path = ROOT / "platforms" + + +_current = [SiteInputs()] + + +def current() -> SiteInputs: + return _current[0] + + +def use(inputs: SiteInputs) -> None: + _current[0] = inputs diff --git a/tests/test_site_inputs.py b/tests/test_site_inputs.py new file mode 100644 index 000000000..ab75d4be7 --- /dev/null +++ b/tests/test_site_inputs.py @@ -0,0 +1,60 @@ +"""generate_site --db reaches every page. + +The run loaded the given database, but the page-finishing pass reloaded the +repository's database.json: a card held only by the given collection stopped +the whole run on a KeyError. +""" + +from __future__ import annotations + +import ast +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import siteinputs # noqa: E402 + +SHA1 = "0123456789ab" + "c" * 28 + + +class _Inputs(unittest.TestCase): + def use(self, **paths: Path) -> None: + siteinputs.use(siteinputs.SiteInputs(**paths)) + self.addCleanup(siteinputs.use, siteinputs.SiteInputs()) + + +class TheGivenDatabase(_Inputs): + def setUp(self) -> None: + tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None)) + self.addCleanup(shutil.rmtree, tmp) + self.db = tmp / "database.json" + self.db.write_text(json.dumps({"files": {SHA1: {"path": "bios/X/only-here.bin"}}})) + + def test_a_card_held_elsewhere_resolves_in_the_given_collection(self) -> None: + self.use(db=self.db) + page = generate_site._cards_elsewhere("[x](#sha1-0123456789ab) ") + self.assertIn(f"files/?sha1={SHA1}", page) + + def test_paths_are_read_from_the_given_collection(self) -> None: + self.use(db=self.db) + self.assertEqual(generate_site._collection_sha1_by_path(), {"bios/X/only-here.bin": SHA1}) + + +class GenerateInstallsTheInputs(unittest.TestCase): + def test_generate_installs_the_inputs_first(self) -> None: + source = ast.get_source_segment( + (REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8"), + next(n for n in ast.parse((REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")).body + if isinstance(n, ast.FunctionDef) and n.name == "_generate")) + self.assertLess(source.index("siteinputs.use("), source.index("load_database(")) + + +if __name__ == "__main__": + unittest.main()