feat: round 4 reference and data

This commit is contained in:
Abdessamad Derraz committed 2026-10-23 19:23:58 +02:00
commit 8f50015a0c
15 files changed
+10306 -100

No files matched your search

+6
View File
@@ -8,6 +8,7 @@ import sqlite3
import sys
import tempfile
import unittest
from unittest import mock
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
@@ -138,6 +139,7 @@ class SiteReferenceContracts(unittest.TestCase):
}
return db, coverages, profiles, stats
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_source_references_are_revision_pinned_links(self):
_, _, profiles, _ = self._fixtures()
rendered = _source_ref_markdown(
@@ -168,6 +170,7 @@ class SiteReferenceContracts(unittest.TestCase):
prefixed = _source_ref_markdown(profile, "example/jg.c:213-214")
self.assertEqual(prefixed, "`example/jg.c:213-214`")
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_a_single_repository_links_the_path_as_written(self):
profile = {"source": "https://github.com/PCSX2/pcsx2", "source_commit": "c" * 40}
rendered = _source_ref_markdown(profile, "pcsx2/ImGui/FullscreenUI.cpp:755-759")
@@ -590,6 +593,7 @@ class OnlyAPinnedProfileClaimsAPin(unittest.TestCase):
class ADataDirectorySaysWhatItIs(unittest.TestCase):
"""The page said `dolphin-sys` > `Sys` and nothing of the tree it packs."""
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_description_origin_mode_and_source(self):
from generate_site import generate_emulator_page # noqa: PLC0415
@@ -773,6 +777,7 @@ class ProseCitationsLinkToThePin(unittest.TestCase):
"source_commit": "76c7a84f1f7e80f3e6f2bba96fe100cb24e99124",
"files": [{"name": "x", "source_ref": "libretro.c:10"}]}
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_a_rooted_or_structured_path_is_linked(self):
from sitesources import link_prose_citations
@@ -799,6 +804,7 @@ class ACitationSaysWhoseCodeItIs(unittest.TestCase):
"upstream": "https://github.com/alekmaul/a5200",
"source_commit": "a" * 40, "upstream_commit": "b" * 40}
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_each_part_links_to_its_repository_with_its_label(self):
from sitesources import _source_ref_markdown, requested_by
+2 -2
View File
@@ -208,7 +208,7 @@ class PlatformReferenceInWords(unittest.TestCase):
systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}}
page = self._page("md5", "md5", systems)
name = siterender.system_name("sony-playstation")
self.assertIn(f"| [{name}](#sony-playstation) <small>sony-playstation</small> |", page)
self.assertIn(f"| [{name}](#sony-playstation) <small data-pagefind-ignore>sony-playstation</small> |", page)
heading = next(line for line in page.splitlines() if line.startswith("### ")
and "sony-playstation" in line)
self.assertIn(name, heading)
@@ -335,7 +335,7 @@ class DatesAndNames(unittest.TestCase):
'<time datetime="2026-08-07">7 August 2026</time>')
def test_data_page_title(self):
self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n"))
self.assertIn("\n---\n\n# Data and API\n", generate_site.generate_data_page([]))
@unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(),
+8 -5
View File
@@ -296,7 +296,7 @@ class ManifestsArePublished(unittest.TestCase):
self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page)
self.assertIn("## Platform manifests", page)
self.assertIn("**Licence:**", page)
self.assertIn("## Hosting and limits", page)
self.assertIn("## Hosting, rate limits and caching", page)
# The endpoints first, the DAT below them.
self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML"))
@@ -449,7 +449,7 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase):
def tearDown(self):
sitesources._tree_paths.cache_clear()
def _render(self, tree: frozenset[str] | None) -> str:
def _render(self, tree: frozenset[str]) -> str:
original = sitesources._tree_paths
sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment]
try:
@@ -464,9 +464,12 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase):
self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text)
def test_without_a_cached_tree_the_declarations_decide(self):
text = self._render(None)
self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text)
def test_a_path_no_tree_confirmed_is_not_linked(self):
"""Round 4: without a tree the declared repository took the path,
and the deploy, which has no tree cache, linked 30 paths to a 404."""
text = self._render(frozenset())
self.assertNotIn("](https://", text)
self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
def test_every_rendered_link_is_in_its_cached_tree(self):
"""Offline: each link the pages render points at a path the cached
+344
View File
@@ -0,0 +1,344 @@
"""Reference pages and data (round 4, package 8).
The lookup answered a SHA1 without the setups that list the same file by
MD5, the published folder check failed 134 EmuDeck entries a folder EmuDeck
accepts, the manifest table printed `&lt;ref>` behind a dead link and showed
17 of 48 fields, 28 system ids held files and no page, and source links went
to 404s on the deploy, which has no tree cache. Tests that read the built
site skip when it has not been generated.
"""
from __future__ import annotations
import hashlib
import json
import re
import sys
import tempfile
import unittest
import urllib.parse
from pathlib import Path
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import check_platform_folder # noqa: E402
import generate_site # noqa: E402
import release_record # noqa: E402
import sitesources # noqa: E402
import upstream # noqa: E402
PUBLIC = REPO_ROOT / "website" / "public"
API = PUBLIC / "api" / "v1"
DOCS = REPO_ROOT / "website" / "src" / "content" / "docs"
DIST = REPO_ROOT / "website" / "dist"
TMP = REPO_ROOT / "tmp"
def _built(path: Path) -> Path:
if not path.exists():
raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated")
return path
def _page(stem: str) -> str:
for suffix in (".md", ".mdx"):
path = DOCS / f"{stem}{suffix}"
if path.is_file():
return path.read_text(encoding="utf-8")
raise unittest.SkipTest(f"{stem} not generated")
class LookupRecordsUnderEveryHash(unittest.TestCase):
"""8.1: a platform listing a file by MD5 is found by its SHA1 too."""
SHA1 = "a" * 40
FILE = {"name": "bios.bin", "path": "bios/Sony/PlayStation/bios.bin", "sha1": SHA1,
"md5": "b" * 32, "crc32": "c" * 8, "sha256": "d" * 64}
def records(self) -> list[dict]:
db = {"files": {self.SHA1: self.FILE}, "indexes": {
"by_md5": {"b" * 32: self.SHA1}, "by_crc32": {"c" * 8: self.SHA1},
"by_sha256": {"d" * 64: self.SHA1}}}
coverages = {"setup": {"config": {"systems": {"sony-playstation": {"files": [
{"name": "bios.bin", "md5": "b" * 32}]}}}}}
profiles = {"core": {"type": "libretro", "systems": ["sony-playstation"],
"files": [{"name": "bios.bin", "crc32": "c" * 8}]}}
with mock.patch.object(generate_site, "_console_system_names", return_value={}), \
mock.patch.object(generate_site, "_lookup_system_name", return_value=None), \
mock.patch.object(generate_site, "emulator_card_page", side_effect=lambda k, a: k):
return generate_site.file_lookup_records(
db, coverages, profiles, [{"sha1": self.SHA1, "url": "/x"}])
def test_each_hash_names_the_setup_and_the_emulator(self):
records = self.records()
for value in (self.SHA1, "b" * 32, "c" * 8, "d" * 64):
owners = {(r["layer"], r.get("owner")) for r in records if r["value"] == value}
self.assertIn(("platform", "setup"), owners, value)
self.assertIn(("emulator", "core"), owners, value)
def test_records_carry_the_sha1_and_say_what_was_declared(self):
for record in self.records():
if record["layer"] in ("platform", "emulator"):
self.assertEqual(record["sha1"], self.SHA1)
copy = next(r for r in self.records()
if r["layer"] == "platform" and r["value"] == self.SHA1)
self.assertEqual(copy["declared"], "md5")
def test_scph5501_gives_one_answer_for_its_four_hashes(self):
db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
sha1 = "0555c6fae8906f3f09baf5988f00e55f88e9f30b"
entry = db["files"][sha1]
_built(API / "lookup")
answers = []
for algorithm in ("sha1", "md5", "crc32", "sha256"):
value = str(entry[algorithm]).lower()
shard = json.loads((API / "lookup" / f"{value[:2]}.json").read_text())
found = [r for r in shard["items"] if r["value"] == value]
answers.append(tuple(sorted({(r["layer"], r.get("owner", "")) for r in found})))
self.assertEqual(len(set(answers)), 1, answers)
setups = {owner for layer, owner in answers[0] if layer == "platform"}
emulators = {owner for layer, owner in answers[0] if layer == "emulator"}
self.assertGreaterEqual(len(setups), 10)
self.assertGreaterEqual(len(emulators), 8)
class TheFolderCheckReadsEmuDecksWhitelist(unittest.TestCase):
"""8.2: an entry with no destination names no file. EmuDeck's check
passes the system when any top-level file of the folder has its md5."""
def check(self, files: dict[str, bytes]) -> list[tuple[str, str]]:
good = b"ps1 bios bytes"
contract = {"verification_mode": "md5", "systems": {"sony-playstation": {"files": [
{"name": f"sony-playstation:{hashlib.md5(good).hexdigest()}", "destination": "",
"md5": hashlib.md5(good).hexdigest()},
{"name": "sony-playstation:" + "0" * 32, "destination": "", "md5": "0" * 32},
]}}}
with tempfile.TemporaryDirectory(dir=TMP) as tmp:
root = Path(tmp) / "bios"
(root / "sub").mkdir(parents=True)
for name, data in files.items():
(root / name).write_bytes(data)
api = Path(tmp) / "api"
api.mkdir()
(api / "emudeck.json").write_text(json.dumps({"items": [{"contract": contract}]}))
found = check_platform_folder.check("emudeck", root, api.as_uri() + "/")
# The verifier's verdict on the matching entry, file by file.
from verify import verify_entry_md5 # noqa: PLC0415
verdicts = {name: verify_entry_md5(contract["systems"]["sony-playstation"]["files"][0],
str(root / name))["status"]
for name in files if (root / name).is_file()}
return found, verdicts
def test_any_name_passes(self):
found, verdicts = self.check({"my-ps1-dump.bin": b"ps1 bios bytes"})
self.assertEqual(found, [("sony-playstation: any file", "ok")])
self.assertEqual(verdicts, {"my-ps1-dump.bin": "ok"})
def test_a_folder_without_it_is_missing(self):
found, _verdicts = self.check({"other.bin": b"x", "sub/deep.bin": b"ps1 bios bytes"})
self.assertEqual(found, [("sony-playstation: any file", "missing")])
def test_the_data_page_states_the_rule(self):
page = generate_site.generate_data_page([])
self.assertIn("An entry with no `destination` (EmuDeck) names no file", page)
self.assertLess(len((REPO_ROOT / "scripts" / "check_platform_folder.py")
.read_text().splitlines()), 70)
class ManifestFieldsAreComplete(unittest.TestCase):
"""8.3 and 8.4."""
SCHEMA = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text())
def test_a_url_template_is_code(self):
rows = generate_site._schema_field_rows(self.SCHEMA["$defs"]["file"], ("repo_path",))
self.assertIn("`https://raw.githubusercontent.com/Abdess/retrobios/<ref>/<repo_path>`",
rows[0])
rows = generate_site._schema_field_rows(
{"properties": {"a": {"description": "From https://x.test/<asset>, then."}}}, ("a",))
self.assertEqual(rows, ["| `a` | From `https://x.test/<asset>`, then. |"])
def test_every_field_is_described_and_shown(self):
page = generate_site.generate_data_page([])
for schema in (self.SCHEMA, self.SCHEMA["$defs"]["file"]):
for key, spec in schema["properties"].items():
self.assertTrue(spec.get("description"), key)
self.assertIn(f"| `{key}` |", page)
def test_no_link_carries_an_escaped_bracket(self):
html = _built(DIST / "data" / "index.html").read_text(encoding="utf-8")
self.assertNotRegex(html, r'href="[^"]*&amp;lt;')
self.assertNotRegex(html, r'href="[^"]*&lt;')
class EverySystemIdWithFilesHasAPage(unittest.TestCase):
"""8.5: tyrian, scummvm, dos and 25 more had files and a null url."""
def test_built_index(self):
items = json.loads(_built(API / "systems.json").read_text())["items"]
orphans = [item["id"] for item in items if item["sha1"] and not item["url"]]
self.assertEqual(orphans, [])
def test_first_file_then_emulator(self):
files = {"f" * 40: {"path": "bios/DOS/DOS/a.bin"}}
with mock.patch("siterender._card_pages", {}):
place = generate_site._first_file_place(
"dos", ["f" * 40], files, {}, lambda path, sha1: "dos")
self.assertEqual(place, ("DOS", "/retrobios/systems/dos/#sha1-ffffffffffff"))
place = generate_site._first_file_place(
"tyrian", [], files, {"opentyrian": {"systems": ["tyrian"]}}, None)
self.assertEqual(place, ("", "/retrobios/emulators/opentyrian/"))
# A source permalink of each forge family the pages write.
_PERMALINK = re.compile(
r"\]\(https://(?P<host>[^/]+)/(?P<slug>[^/]+/[^/]+)/"
r"(?:blob/|-/blob/|src/commit/)(?P<pin>[0-9a-f]{40})/(?P<path>[^#)]+)")
class SourceLinksOnlyWhereTheFileIs(unittest.TestCase):
"""8.6: the deploy reads the committed record, never a tree cache."""
def test_an_address_in_a_binary_reads_as_one(self):
profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40}
text = sitesources._source_ref_markdown(
profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac")
self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, "
"offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`")
def test_only_a_recorded_path_links(self):
profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40}
repo = upstream.parse_repo("https://github.com/o/r")
record = {sitesources.tree_key(repo, "e" * 40): frozenset({"src/a.c"})}
with mock.patch.object(sitesources, "_verified", return_value=record):
sitesources._tree_paths.cache_clear()
try:
text = sitesources._source_ref_markdown(profile, "src/a.c:5, src/b.c:7")
finally:
sitesources._tree_paths.cache_clear()
self.assertIn("[`src/a.c:5`](https://github.com/o/r/blob/" + "e" * 40 + "/src/a.c#L5)", text)
self.assertIn("`src/b.c:7`", text)
self.assertNotIn("src/b.c#", text)
def test_the_writer_keeps_held_paths_only(self):
profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40,
"notes": "see src/c.c:3", "files": [{"source_ref": "src/a.c:5, gone.c:1"}]}
with mock.patch.object(sitesources, "_cached_tree",
return_value=frozenset({"src/a.c", "src/c.c", "x.c"})):
held, unread = sitesources.build_source_paths({"p": profile}, [TMP])
self.assertEqual(held, {"github.com/o/r@" + "e" * 40: ["src/a.c", "src/c.c"]})
self.assertEqual(unread, [])
def test_every_rendered_link_is_recorded(self):
record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*"))
checked, stray = 0, []
for page in pages:
for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")):
if "/Abdess/retrobios/" in found[0]:
continue
key = f"{found['host']}/{found['slug']}@{found['pin']}"
path = urllib.parse.unquote(found["path"])
checked += 1
if path not in record.get(key, ()):
stray.append((page.name, key, path))
self.assertGreater(checked, 1000)
self.assertEqual(stray[:20], [])
def test_the_record_is_read_by_the_deploy(self):
"""The record is committed where the build reads it, sorted."""
record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
self.assertGreater(len(record), 300)
for key, paths in record.items():
self.assertRegex(key, r"^[a-z0-9.-]+/.+@[0-9a-f]{40}$")
self.assertEqual(paths, sorted(set(paths)), key)
class TheReferencePagesAnswerTheirWords(unittest.TestCase):
"""8.7, 8.8, 8.10 and 8.13."""
def test_find_or_check_a_file(self):
page = generate_site.generate_files_page()
self.assertTrue(page.startswith(
"---\nterms: check dump, md5 check, verify bios file, check hash\n---\n"))
self.assertIn("\n# Find or check a file\n\nCheck a BIOS dump: paste its hash "
"or choose the file.\n", page)
def test_the_data_page(self):
page = generate_site.generate_data_page([])
self.assertTrue(page.startswith(
"---\nterms: manifest, schema, check a folder, rate limit, api\n---\n"))
self.assertIn("## Hosting, rate limits and caching", page)
self.assertIn("HTTP 429", page)
self.assertIn("crypto.subtle.digest(\"SHA-1\"", page)
self.assertIn("api/v1/lookup/", page)
self.assertIn("`preferred: true`", page)
self.assertIn("`database.json`, `stats.json`, `release.json` and the manifests are not "
"envelopes", page)
self.assertIn("- MinUI, MyMinUI and NextUI have no list of their own "
"(`native_list: none`). An empty list does not mean nothing is needed.", page)
def test_overview_ids_and_counts_are_not_indexed(self):
html = _built(DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8")
self.assertRegex(html, r"<small data-pagefind-ignore(=\"true\")?>sony-playstation</small>")
self.assertRegex(html, r"<span data-pagefind-ignore(=\"true\")?>\d+/\d+</span>")
class TheReleaseRecordHasAVersion(unittest.TestCase):
"""8.9."""
def test_the_writer_and_the_committed_record(self):
self.assertEqual(release_record.SCHEMA_VERSION, 1)
source = (REPO_ROOT / "scripts" / "release_record.py").read_text(encoding="utf-8")
self.assertIn('{"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}', source)
schema = json.loads((REPO_ROOT / "schemas" / "release.schema.json").read_text())
self.assertIn("schema_version", schema["required"])
record = json.loads((REPO_ROOT / "release.json").read_text())
self.assertEqual(record["schema_version"], 1)
class NoEmptyCoreTable(unittest.TestCase):
"""8.11: MiSTer and BizHawk declare cores: [] and had a page of nothing."""
def test_own_list_platforms_get_a_sentence(self):
coverages = {"misterfpga": {"platform": "MiSTer FPGA", "config": {"cores": [], "systems": {}}}}
moved: dict[str, str] = {}
page = generate_site.generate_cross_reference(coverages, {}, moved)
self.assertEqual(moved, {})
self.assertIn(generate_site.OWN_LIST_ONLY, page)
def test_built_pages(self):
_built(DOCS / "cross-reference.md")
for name in ("misterfpga", "bizhawk"):
self.assertFalse((DOCS / f"cross-reference-{name}.md").exists())
self.assertIn(generate_site.OWN_LIST_ONLY, _page(f"platforms/{name}"))
redirects = json.loads((REPO_ROOT / "website" / "src" / "generated" /
"redirects.json").read_text())
self.assertEqual(redirects.get("/cross-reference-misterfpga/"), "/platforms/misterfpga/")
class FilesToSourceOnAPhone(unittest.TestCase):
"""8.12: 601 rows on one page, no way to give a file."""
def test_parts_of_a_hundred_rows_each_with_an_upload(self):
_built(DOCS / "gaps-files-to-source.md")
parts = sorted(DOCS.glob("gaps-files-to-source*.md"))
self.assertGreater(len(parts), 1)
for part in parts:
rows = [line for line in part.read_text(encoding="utf-8").splitlines()
if line.startswith("| `")]
self.assertLessEqual(len(rows), generate_site.FILES_TO_SOURCE_PART, part.name)
for row in rows:
self.assertIn(f"[Have it? Upload]({generate_site.UPLOAD_URL}", row)
def test_the_upload_lands_in_the_console_folder(self):
with mock.patch("siterender._own_section", return_value=("Sony", "PlayStation")):
self.assertEqual(generate_site._upload_folder({"systems": ["sony-playstation"]}),
"Sony/PlayStation")
if __name__ == "__main__":
unittest.main()
+2
View File
@@ -113,6 +113,7 @@ class StructuredBlocks(unittest.TestCase):
return "\n".join(generate_site._render_structured_blocks(profile))
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def test_a_source_ref_in_a_block_is_a_pinned_link(self):
"""186 citations in analysis and firmware blocks were plain code."""
page = self.render(self.profile("analysis", {"tmss": {
@@ -237,6 +238,7 @@ class ForgesUpstreamDoesNotRead(unittest.TestCase):
"""Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
Heritage showed no revision, and their citations were plain code."""
@mock.patch("sitesources.tree_holds", lambda *_args: True) # a path the record confirms
def links(self, url: str, ref: str, **extra: str) -> tuple[str, str]:
import sitesources
+1 -1
View File
@@ -41,7 +41,7 @@ class BinaryContinuationsAreNotLinked(unittest.TestCase):
def _render(self, value: str) -> str:
original = sitesources._tree_paths
sitesources._tree_paths = lambda repo, pin: None # type: ignore[assignment]
sitesources._tree_paths = lambda repo, pin: frozenset({"FPLoader.cpp"}) # type: ignore[assignment]
try:
return sitesources._source_ref_markdown(self.PROFILE, value)
finally: