fix: read the given database on every page

The page-finishing pass reloaded the repository's database.json whatever
--db said, so a card held only by the given collection raised KeyError.
siteinputs holds the inputs generate_site was given.
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 08:12:43 +01:00
1 parent 19898cb6ea
commit 8022ef4cb3
3 files changed
+110 -7

No files matched your search

+18 -7
View File
@@ -89,6 +89,7 @@ from generate_readme import (
from provenance_report import build_report, distinct_missing
import sitemarkup as markup
import siteentries
import siteinputs
import retired_files
import sitestarlight
from siteexports import generate_exports_page, load_export_reports
@@ -3174,7 +3175,9 @@ def _front_matter(markdown: str) -> dict:
return data if isinstance(data, dict) else {}
_SHA1_BY_PREFIX: dict[str, str] = {}
@functools.lru_cache(maxsize=None)
def _sha1_by_prefix(db_path: str) -> dict[str, str]:
return {sha1[:12]: sha1 for sha1 in load_database(db_path)["files"]}
def _cards_elsewhere(markdown: str) -> str:
@@ -3185,11 +3188,9 @@ def _cards_elsewhere(markdown: str) -> str:
away = {ident for ident in links if f'id="{ident}"' not in markdown}
if not away:
return markdown
if not _SHA1_BY_PREFIX:
db = load_database(str(_REPO_ROOT / "database.json"))
_SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]})
by_prefix = _sha1_by_prefix(str(siteinputs.current().db))
for ident in away:
full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")]
full = by_prefix[ident.removeprefix("sha1-")]
markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})")
return markdown
@@ -3500,10 +3501,14 @@ def _list_pages(
f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1))
@functools.lru_cache(maxsize=1)
def _collection_sha1_by_path() -> dict[str, str]:
"""The sha1 of each file the collection holds, by its path."""
with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
return _sha1_by_path_in(str(siteinputs.current().db))
@functools.lru_cache(maxsize=None)
def _sha1_by_path_in(db_path: str) -> dict[str, str]:
with open(db_path, encoding="utf-8") as handle:
files = json.load(handle).get("files", {})
return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()}
@@ -12260,6 +12265,12 @@ def main() -> None:
def _generate(args: argparse.Namespace) -> None:
# Every page reads these, never the repository's own files.
siteinputs.use(siteinputs.SiteInputs(
db=Path(args.db).resolve(),
emulators=Path(args.emulators_dir).resolve(),
platforms=Path(args.platforms_dir).resolve(),
))
db = load_database(args.db)
docs = Path(args.docs_dir)
# Files the site serves as they are (data, downloads, images), in the
+32
View File
@@ -0,0 +1,32 @@
"""The collection, profiles and platform files the site describes.
generate_site.py takes them as --db, --emulators-dir and --platforms-dir and
installs them here before writing a page. Every page module reads its inputs
from current(), so a run on another collection describes that collection on
every page. A module used on its own reads the repository's files.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
@dataclass(frozen=True)
class SiteInputs:
db: Path = ROOT / "database.json"
emulators: Path = ROOT / "emulators"
platforms: Path = ROOT / "platforms"
_current = [SiteInputs()]
def current() -> SiteInputs:
return _current[0]
def use(inputs: SiteInputs) -> None:
_current[0] = inputs
+60
View File
@@ -0,0 +1,60 @@
"""generate_site --db reaches every page.
The run loaded the given database, but the page-finishing pass reloaded the
repository's database.json: a card held only by the given collection stopped
the whole run on a KeyError.
"""
from __future__ import annotations
import ast
import json
import shutil
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site # noqa: E402
import siteinputs # noqa: E402
SHA1 = "0123456789ab" + "c" * 28
class _Inputs(unittest.TestCase):
def use(self, **paths: Path) -> None:
siteinputs.use(siteinputs.SiteInputs(**paths))
self.addCleanup(siteinputs.use, siteinputs.SiteInputs())
class TheGivenDatabase(_Inputs):
def setUp(self) -> None:
tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None))
self.addCleanup(shutil.rmtree, tmp)
self.db = tmp / "database.json"
self.db.write_text(json.dumps({"files": {SHA1: {"path": "bios/X/only-here.bin"}}}))
def test_a_card_held_elsewhere_resolves_in_the_given_collection(self) -> None:
self.use(db=self.db)
page = generate_site._cards_elsewhere("[x](#sha1-0123456789ab) ")
self.assertIn(f"files/?sha1={SHA1}", page)
def test_paths_are_read_from_the_given_collection(self) -> None:
self.use(db=self.db)
self.assertEqual(generate_site._collection_sha1_by_path(), {"bios/X/only-here.bin": SHA1})
class GenerateInstallsTheInputs(unittest.TestCase):
def test_generate_installs_the_inputs_first(self) -> None:
source = ast.get_source_segment(
(REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8"),
next(n for n in ast.parse((REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")).body
if isinstance(n, ast.FunctionDef) and n.name == "_generate"))
self.assertLess(source.index("siteinputs.use("), source.index("load_database("))
if __name__ == "__main__":
unittest.main()