chore: merge audit fixes (site)

# Conflicts:
#	emulators/flycast.yml
#	emulators/primehack.yml
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 11:48:30 +02:00
commit 8643d4a4ea
50 files changed
+1352 -176

No files matched your search

+61
View File
@@ -1700,6 +1700,49 @@ class TestBuildReport(unittest.TestCase):
report = build_report("test", self._versioned("v0.78"), self.dir)
self.assertEqual(report.pinned_tag, "0.78")
def test_an_upstream_on_its_release_tag_does_not_freeze_the_port(self):
"""GroovyMAME declares 0.289 and its upstream_commit is the mame0289
tag. Reading that as a frozen profile stopped every comparison and
bump of the port, which keeps moving. The upstream is read at its tag,
the port against its HEAD."""
self.files[("portpin", "src/p.c")] = ["x", "load p.bin", "y"]
self.files[("porthead", "src/p.c")] = ["x", "load p.bin", "y"]
self.files[("uppin", "src/u.c")] = ["x", "ROM_LOAD(u.bin)", "y"]
self.files[("uphead", "src/u.c")] = ["x", "", "", "", "y"]
heads = {"o/port": "porthead", "o/up": "uphead"}
profile_sync.upstream.resolve_head = (
lambda repo, cache, offline=False, branch=None: heads[repo.slug]
)
# A tag off the tracked branch is not an orphaned pin.
profile_sync.upstream.compare = (
lambda repo, base, head, cache_dir, offline=False: CompareResult(
[], False, repo.slug == "o/up"
)
)
self.tag_commits = {"0.289": "uppin"}
profile = {
"emulator": "Port",
"source": "https://github.com/o/port",
"upstream": "https://github.com/o/up",
"source_commit": "portpin",
"upstream_commit": "uppin",
"core_version": "0.289",
"files": [
{"name": "p.bin", "source_ref": "source src/p.c:2"},
{"name": "u.bin", "source_ref": "upstream src/u.c:2"},
],
}
report = build_report("port", profile, self.dir)
self.assertIsNone(report.pinned_tag)
self.assertEqual(report.upstream_tag, "0.289")
self.assertEqual(report.orphaned, [])
self.assertEqual(
[entry.status for entry in report.entries], ["ANCHORED", "ANCHORED"]
)
self.assertIn(
"upstream pinned to tag 0.289", profile_sync.format_report(report)
)
def test_a_tag_pointing_elsewhere_is_not_flagged(self):
self.files[("pinsha", "a.c")] = ["x", "hit"]
self.files[("headsha", "a.c")] = ["x", "hit"]
@@ -2983,6 +3026,24 @@ class TestBumpCommit(unittest.TestCase):
self.assertEqual(rebase_refs(self.path, report), ["a.c:10-12 -> a.c:30-32"])
self.assertTrue(bump_commit(self.path, report))
def test_an_upstream_on_its_release_tag_stays_there(self):
"""GroovyMAME's upstream_commit is the mame0289 tag it merged: a bump
of the port must not carry it to mamedev HEAD, another program."""
part = PartResult(
RefPart("a.c", 10, 12, "a.c:10-12"), "ANCHORED", None, 10, 12, []
)
report = ProfileReport(
name="p", repo="o/n", pin="pin", head="newhead",
entries=[EntryReport("a.bin", "a.c:10-12", "ANCHORED", [part])],
counts={"ANCHORED": 1},
upstream_pin="uptag", upstream_head="uphead",
upstream_origin="upstream_commit", upstream_tag="mame0289",
)
self.assertTrue(bump_commit(self.path, report))
written = yaml.safe_load(self.path.read_text())
self.assertEqual(written["source_commit"], "newhead")
self.assertNotIn("upstream_commit", written)
def test_upstream_pin_moves_with_source_pin(self):
"""A recale onto upstream HEAD lines must advance upstream_commit too."""
part = PartResult(
+52
View File
@@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase):
self.assertTrue(available({"name": "pak0.pk3"}))
self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"}))
def test_a_release_asset_is_held_only_when_collected(self):
"""storage: large_file said "in repo" for kenji-nx's user-picked
firmware package, which the collection does not hold."""
available = _availability_check(DB, set())
self.assertFalse(available({"name": "<firmware>.zip", "storage": "large_file"}))
self.assertTrue(available({"name": "pak0.pk3", "storage": "release"}))
def test_the_badge_says_why(self):
entry = {"name": "pak0.pk3", "unsourceable": "retail data"}
badges = " ".join(_file_badges(entry, False))
@@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase):
self.assertEqual(shown_held, [])
class AHeldEntrySaysWhere(unittest.TestCase):
"""164 entries wore the "in repo" badge with no copy named: they are held
as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a
data directory (Dolphin's fonts), which the database does not index."""
def test_member_file_and_tree(self):
import hashlib
import tempfile
import zipfile
import generate_site
from cross_reference import FileConsumers, held_copy
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
data, bios = Path(tmp) / "data", Path(tmp) / "bios"
(data / "dolphin-sys" / "GC").mkdir(parents=True)
(data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font")
(bios / "Arcade" / "MAME").mkdir(parents=True)
archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip"
with zipfile.ZipFile(archive, "w") as zf:
zf.writestr("specmate.rom", b"rom")
roots = {"data_root": str(data), "bios_root": str(bios)}
member = held_copy({"name": "specmate.rom",
"sha1": hashlib.sha1(b"rom").hexdigest()}, **roots)
self.assertEqual(member, (str(archive), "specmate.rom"))
# A declared hash the copy does not carry names no copy.
self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots))
self.assertEqual(held_copy({"name": "font_western.bin"}, **roots),
(str(data / "dolphin-sys/GC/font_western.bin"), ""))
self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots),
(f"{(data / 'dolphin-sys/GC').as_posix()}/", ""))
consumers = FileConsumers({"files": {"z" * 40: {
"path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={})
line = generate_site._held_copy_line(
("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers)
self.assertEqual(line, "Collection copy: member `specmate.rom` of "
"[`bios/Arcade/MAME/spectrum_opus.zip`]"
"(../systems/arcade.md#sha1-zzzzzzzzzzzz)")
line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""),
consumers)
self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the "
"`dolphin-sys` data directory")
if __name__ == "__main__":
unittest.main()
+142
View File
@@ -0,0 +1,142 @@
"""What an emulator page says about a profile and its files.
No network and no database: pages are rendered from small profiles.
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site as gs # noqa: E402
EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}}
def _page(name: str, profile: dict) -> str:
return gs.generate_emulator_page(name, profile, EMPTY_DB, None, set())
class WhereAndHowAFileIsRead(unittest.TestCase):
"""load_from, found_by and on_mismatch were in 99, 4 and 25 entries and
on no page: Dolphin's gba_bios.bin showed a Path a RetroArch user put
under system/, while the libretro core reads it from its save directory."""
def test_load_from(self):
page = _page("dolphin", {"type": "standalone + libretro", "files": [
{"name": "gba_bios.bin", "path": "GBA/gba_bios.bin",
"load_from": {"libretro": "save_dir"}},
{"name": "hdd.rom", "load_from": "hdd_image"},
{"name": "bios.bin", "load_from": "system_dir"},
]})
self.assertIn("- Read from: the save directory, not the BIOS directory "
"(libretro build)", page)
self.assertIn("- Read from: inside the emulated hard disk image", page)
self.assertEqual(page.count("Read from:"), 2)
def test_found_by_and_on_mismatch(self):
page = _page("x", {"type": "standalone", "found_by": "content", "files": [
{"name": "scph5501.bin", "found_by": "content", "on_mismatch": "warn"},
{"name": "boot.bin", "on_mismatch": {"signature": "warn"}},
]})
self.assertIn("| Files found by | content (the code scans the folder", page)
self.assertIn("- Found by: content (", page)
self.assertIn("- On a failed check: warns and runs with the file", page)
self.assertIn("- On a failed signature check: warns and runs with the file", page)
class WhoShipsTheBuildAndWhy(unittest.TestCase):
"""built_by, pin_frozen and version_source were in 9, 13 and 40 profiles
and on no page: mame0278 did not say only Recalbox ships it."""
def test_meta_rows(self):
page = _page("mame0278", {
"type": "libretro", "built_by": ["recalbox"],
"pin_frozen": "ROCKNIX 20261001 builds this revision.",
"version_source": {"url": "https://example.org/news", "pattern": "v(\\S+)",
"no_genuine_copy": "13.0"},
"files": [],
})
self.assertIn("| Built by | [recalbox](../platforms/recalbox.md), under its "
"core name.", page)
self.assertIn("| Revision held | ROCKNIX 20261001 builds this revision. |", page)
self.assertIn("| Releases announced at | [https://example.org/news]"
"(https://example.org/news) (version 13.0 announced there, "
"found only as rebuilt copies) |", page)
class ACardKeepsItsAddress(unittest.TestCase):
"""emulators/raze/#file-duke3d-grp stopped pointing at its card once a
second version of the path was declared: the anchor of the first one
became file-duke3d-grp-<hash>."""
def test_a_second_version_moves_nothing(self):
one = {"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"}
two = {"name": "DUKE3D.GRP", "md5": "cccc2222dddd"}
before = gs.emulator_file_anchors([one])
after = gs.emulator_file_anchors([two, one])
self.assertEqual(before, ["file-duke3d-grp-aaaa1111"])
self.assertEqual(after[1], before[0])
def test_the_plain_address_still_lands(self):
files = [{"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"}]
page = _page("raze", {"type": "standalone", "files": files})
self.assertIn('<span id="file-duke3d-grp"></span>', page)
self.assertIn("{ #file-duke3d-grp-aaaa1111 .rb-file-title }", page)
# A card already holding the plain id gives it to no one else.
files = [{"name": "x.bin"}, {"name": "x.bin", "sha1": "ab" * 20}]
anchors = gs.emulator_file_anchors(files)
self.assertEqual(anchors, ["file-x-bin", "file-x-bin-abababab"])
self.assertEqual(gs.emulator_file_aliases(files, anchors), ["", ""])
class WhatASearchSnippetSays(unittest.TestCase):
def test_a_core_loading_no_file_promises_no_hash(self):
"""152 pages of cores that load no file announced paths and hashes."""
rel = Path("emulators/lutro.md")
saved = dict(gs.PAGE_DATA)
self.addCleanup(lambda: (gs.PAGE_DATA.clear(), gs.PAGE_DATA.update(saved)))
gs.PAGE_DATA[rel.as_posix()] = {}
text = gs._page_description("# Lutro\n", rel, "Lutro")
self.assertNotIn("hashes", text)
self.assertIn("No BIOS or firmware file is required.", text)
gs.PAGE_DATA[rel.as_posix()] = {"hasPart": [{"name": "a.bin"}]}
self.assertIn("paths, hashes", gs._page_description("# Lutro\n", rel, "Lutro"))
def test_a_long_paragraph_is_cut_at_a_word(self):
plain = ("files/?sha1= is the address that does not move: it opens the card "
"wherever the file is now, " * 4)
text = gs._shorten(plain)
self.assertTrue(text.endswith("..."))
self.assertLessEqual(len(text), 160)
self.assertTrue(plain.startswith(text[:-3]))
self.assertIn(plain[len(text) - 3], " ,:")
def test_code_keeps_its_underscores(self):
self.assertEqual(gs._plain_markdown("only `validate_schemas.py` and _this_"),
"only validate_schemas.py and this")
def test_structured_url_is_encoded_like_the_canonical(self):
"""mkdocs encodes idtech4a++ as idtech4a%2B%2B in the canonical link
and the sitemap, and the JSON-LD named the unencoded URL."""
import json
import tempfile
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
page = Path(tmp) / "emulators" / "idtech4a++.md"
page.parent.mkdir()
page.write_text("# idTech4A++\n\nSome text long enough to be a description.\n")
gs.decorate_markdown_pages(Path(tmp))
text = page.read_text()
data = json.loads(text.split('<script type="application/ld+json">')[1]
.split("</script>")[0].replace("<\\/", "</"))
self.assertEqual(data["url"],
"https://abdess.github.io/retrobios/emulators/idtech4a%2B%2B/")
if __name__ == "__main__":
unittest.main()
+120
View File
@@ -882,5 +882,125 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase):
self.assertIn("name: a.bin, size: 2", html)
class TheOneLinerIsWhereUsersLand(unittest.TestCase):
"""The download page had no Android line and no Windows form with
options, and the platform and system pages offered only ZIP parts."""
def test_download_page(self):
import generate_site # noqa: PLC0415
page = generate_site.generate_which_pack()
self.assertIn("Android (Termux", page)
self.assertIn("install.ps1))) --check", page)
self.assertIn("--platform retroarch --dest", page)
def test_platform_and_system_pages(self):
import generate_site # noqa: PLC0415
from cross_reference import FileConsumers # noqa: PLC0415
cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0,
"verified": 0, "missing": 0, "details": [], "config": {"systems": {}}}
page = generate_site.generate_platform_page("batocera", cov)
self.assertIn("install.sh | sh -s -- --platform batocera", page)
self.assertIn("install.ps1))) --platform batocera", page)
self.assertLess(page.index("--platform batocera"), page.index("md-button"))
entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3,
"path": "bios/Recalbox/MSX/a.bin"}
page = generate_site.generate_system_page(
"Microsoft", {"MSX": [entry]}, FileConsumers({"files": {}}),
{"MSX": [("recalbox", "msx1,msx2,msxturbor"), ("batocera", "msx1")]})
self.assertIn("--platform recalbox --system msx1,msx2,msxturbor\n", page)
self.assertIn("--platform batocera --system msx1", page)
def test_a_system_is_offered_only_where_the_manifest_records_it(self):
import generate_site # noqa: PLC0415
import siterender # noqa: PLC0415
saved = dict(siterender._system_page_map)
self.addCleanup(siterender.set_system_page_map, saved)
siterender.set_system_page_map({"sony-playstation": ("sony", "PlayStation"),
"nowhere": ("sony", "PlayStation")})
cov = {"retroarch": {"config": {"systems": {"sony-playstation": {}, "nowhere": {}}}}}
choices = generate_site.system_install_choices(cov)
self.assertEqual(choices["sony"]["PlayStation"], [("retroarch", "sony-playstation")])
class TheGapPageOpensOnCounts(unittest.TestCase):
"""The page opened on a slogan and on tiles reading "Verified (100%)"
for 5,437 of 5,438 files and "Missing 0", above a table of red cells."""
def test_every_count_is_said(self):
import generate_site # noqa: PLC0415
text = generate_site._platform_lists_sentence(5438, 5437, 1, 0)
self.assertIn("Of the 5,438 files", text)
self.assertIn("5,437 are held", text)
self.assertIn("1 is held but not verified", text)
self.assertIn("0 are missing", text)
self.assertNotIn("%", text)
source = (Path(__file__).resolve().parents[1] / "scripts" / "generate_site.py").read_text()
self.assertNotIn("Unified view of BIOS verification", source)
class WhichPackDescribesNoPlatformByOpinion(unittest.TestCase):
"""The page called RetroPie the largest community with the most guides,
Batocera an easy setup and Recalbox good for a first build: claims
nothing in the repository measures, ranking platforms against each
other."""
def test_no_ranking_words(self):
import re
import generate_site # noqa: PLC0415
page = generate_site.generate_which_pack()
rows = [line for line in page.splitlines() if line.startswith("| ")]
found = [
row for row in rows
if re.search(r"\b(largest|most|popular|easy|best|classic|lightweight|"
r"one-click|plug-and-play|first build|accuracy-focused)\b",
row, re.IGNORECASE)
]
self.assertEqual(found, [])
class EveryPageLinksItsOwnData(unittest.TestCase):
"""An emulator page linked its YAML and JSON. A platform page linked
nothing, and a tool had to download platforms.json (2.8 MB) and filter
it. A system page linked nothing either."""
def test_platform_and_system_documents(self):
import generate_site # noqa: PLC0415
from cross_reference import FileConsumers # noqa: PLC0415
cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0,
"verified": 0, "missing": 0, "details": [], "config": {"systems": {}}}
page = generate_site.generate_platform_page("batocera", cov)
self.assertIn("[JSON](../api/v1/platforms/batocera.json)", page)
self.assertIn("/blob/main/platforms/batocera.yml)", page)
entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3,
"path": "bios/Sony/PlayStation/a.bin"}
page = generate_site.generate_system_page(
"Sony", {"PlayStation": [entry]}, FileConsumers({"files": {}}))
self.assertIn("[JSON](../api/v1/systems/sony.json)", page)
item = generate_site.system_export_item("sony", "Sony", {"PlayStation": [entry]})
self.assertEqual(item["consoles"]["PlayStation"][0]["sha1"], "a" * 40)
with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp:
path = Path(tmp) / "sony.json"
generate_site.write_single_item(path, "systems", item, "2026-10-21T00:00:00Z")
document = json.loads(path.read_text())
self.assertEqual(document["count"], 1)
try:
import jsonschema # noqa: PLC0415
except ImportError:
return
root = Path(__file__).resolve().parents[1]
envelope = json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text())
jsonschema.validate(document, envelope)
if __name__ == "__main__":
unittest.main()
+26
View File
@@ -44,5 +44,31 @@ class EachRowItsDeclaration(unittest.TestCase):
self.assertNotIn(A, saturn)
class AnUntestedRowNamesBothHashes(unittest.TestCase):
"""The gaps page said "declared hash contradicted by the local file" for
RetroDECK's dsifirmware.bin and named neither the dump RetroDECK asks for
nor the one the collection holds."""
def test_declared_and_held(self):
import generate_site as gs
path = "bios/Nintendo/Nintendo DS/dsifirmware.bin"
declared, held = "1" * 64, "2" * 64
config = {"systems": {"nintendo-ds": {"files": [
{"name": "dsifirmware.bin", "sha256": declared}]}}}
detail = {"name": "dsifirmware.bin", "system": "nintendo-ds"}
problem = {
"platform": "RetroDECK", "name": "dsifirmware.bin", "status": "untested",
"required": True, "reason": "declared hash contradicted by the local file",
"declared": gs._declared_hashes(config, detail), "path": path,
}
db = {"files": {"f" * 40: {"path": path, "sha256": held}}}
table = "\n".join(gs._render_problem_files([problem], db=db))
row = next(line for line in table.splitlines() if "dsifirmware" in line)
self.assertIn(f"sha256 `{declared}`", row)
self.assertIn(f"sha256 `{held}`", row)
self.assertIn(f"`{path}`", row)
if __name__ == "__main__":
unittest.main()
+83
View File
@@ -147,6 +147,89 @@ class AnIdNamesOneCardPerPage(unittest.TestCase):
self.assertEqual(page.count("rb-sys-file"), 2)
class ACopyFiledByEmulatorDoesNotMoveAMachine(unittest.TestCase):
"""The database keeps one path per content, and the system links followed
it: nintendo-nes and nintendo-fds led to Arcade > FBNeo, where
disksys.rom is kept as fdsbios.nes, sega-mastersystem and
sinclair-zx-spectrum to Other > zesarux."""
def _db(self, tmp: str) -> dict:
for rel in ("Arcade/FBNeo/fdsbios.nes", "Nintendo/Famicom Disk System/disksys.rom",
"Nintendo/NES/other.rom"):
(Path(tmp) / rel).parent.mkdir(parents=True, exist_ok=True)
(Path(tmp) / rel).write_bytes(b"x")
return {
"bios_dir": tmp,
"files": {
"d" * 40: {"path": "bios/Arcade/FBNeo/fdsbios.nes", "sha1": "d" * 40,
"name": "fdsbios.nes"},
"e" * 40: {"path": "bios/Nintendo/NES/other.rom", "sha1": "e" * 40,
"name": "other.rom"},
},
"indexes": {
"by_sha1": {}, "by_md5": {}, "by_name": {},
"by_path_suffix": {
"Arcade/FBNeo/fdsbios.nes": ["d" * 40],
"Nintendo/Famicom Disk System/disksys.rom": ["d" * 40],
"Nintendo/NES/other.rom": ["e" * 40],
},
},
}
def test_the_copy_under_the_declared_name_places_the_system(self):
import tempfile
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
db = self._db(tmp)
files = [{"name": "disksys.rom", "sha1": "d" * 40}]
self.assertEqual(
gs._system_folder(files, db, {}, "nintendo-fds"),
("Nintendo", "Famicom Disk System"),
)
# A file named otherwise still finds the maker's copy before the
# emulator's folder.
files = [{"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}]
self.assertEqual(
gs._system_folder(files, db, {}, "fds"),
("Nintendo", "Famicom Disk System"),
)
def test_an_id_that_spells_a_folder_links_to_it(self):
import tempfile
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
db = self._db(tmp)
manufacturers = gs._group_by_manufacturer(db)
config = {"systems": {"nintendo-nes": {"files": [
{"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}]}}}
pages = gs._system_pages_from_data(
manufacturers, {"p": {"config": config}}, db)
self.assertEqual(pages["nintendo-nes"], ("nintendo", "NES"))
def test_repository_systems(self):
if not (REPO_ROOT / "database.json").exists():
self.skipTest("database.json is not built")
previous = os.getcwd()
os.chdir(REPO_ROOT)
self.addCleanup(os.chdir, previous)
from common import load_database, load_emulator_profiles
db = load_database("database.json")
manufacturers = gs._group_by_manufacturer(db)
pages = gs._system_pages_from_data(
manufacturers, {}, db, load_emulator_profiles("emulators", skip_aliases=False))
expected = {
"nintendo-nes": ("nintendo", "NES"),
"nintendo-fds": ("nintendo", "Famicom Disk System"),
"sega-mastersystem": ("sega", "Master System"),
"sinclair-zx-spectrum": ("sinclair", "ZX Spectrum"),
"commodore-vic20": ("commodore", "VIC-20"),
}
for sys_id, page in expected.items():
with self.subTest(system=sys_id):
self.assertEqual(pages.get(sys_id), page)
class SystemEmulatorsWhenNoIdMatches(unittest.TestCase):
"""RetroArch files NAOMI and Atomiswave as sega-dreamcast-arcade and the
Enterprise as enterprise-64-128, ids no profile uses: the emulator cell