mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-29 23:27:19 -05:00
refactor: compute coverage once, index proofs
EntryCategories asked every core in turn: 3.8 million proofs for 7,430 entries. ProofIndex looks each key up once, and verify --all --json is byte-identical (24 s to 17 s). generate_site and generate_readme now share one coverage, cached in tmp/coverage-cache keyed by its inputs, so the README step drops from 25 s to 3 s.
This commit is contained in:
1 parent
f67f4ead1b
commit
baf38ffbee
6 files changed
+349
-33
No files matched your search
@@ -0,0 +1,100 @@
|
||||
"""The site and the README share one coverage computation.
|
||||
|
||||
generate_site.py and generate_readme.py each verified every platform, a
|
||||
minute of the site build spent twice on the same answer. The first to run
|
||||
stores the coverage, keyed by its inputs, and the second reads it. A cache
|
||||
that outlived a change to its inputs would publish stale counts, so these
|
||||
tests change an input and expect a fresh computation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import generate_readme # noqa: E402
|
||||
|
||||
|
||||
class CoverageCache(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.work = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
|
||||
self.platforms = self.work / "platforms"
|
||||
self.platforms.mkdir()
|
||||
(self.platforms / "alpha.yml").write_text("platform: Alpha\n")
|
||||
self.emulators = self.work / "emulators"
|
||||
self.emulators.mkdir()
|
||||
self.db = self.work / "database.json"
|
||||
self.db.write_text("{}")
|
||||
self.calls: list[str] = []
|
||||
patches = [
|
||||
mock.patch.object(generate_readme, "COVERAGE_CACHE_DIR", self.work / "cache"),
|
||||
mock.patch.object(generate_readme, "compute_coverage", self._compute),
|
||||
]
|
||||
for patch in patches:
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(self.work, ignore_errors=True)
|
||||
|
||||
def _compute(self, name, platforms_dir, db, data_registry=None, emulators_dir=""):
|
||||
self.calls.append(name)
|
||||
if name == "ghost":
|
||||
raise FileNotFoundError(name)
|
||||
return {"platform": name.title(), "missing_names": {"a.bin"}}
|
||||
|
||||
def _run(self, names=("beta", "alpha")) -> dict:
|
||||
return generate_readme.platform_coverages(
|
||||
list(names), str(self.platforms), {}, {},
|
||||
emulators_dir=str(self.emulators), db_path=str(self.db),
|
||||
)
|
||||
|
||||
def test_second_run_reads_the_first(self):
|
||||
first = self._run()
|
||||
self.assertEqual(self.calls, ["alpha", "beta"])
|
||||
second = self._run()
|
||||
self.assertEqual(self.calls, ["alpha", "beta"])
|
||||
self.assertEqual(first, second)
|
||||
# The caller's order, not the computation's.
|
||||
self.assertEqual(list(second), ["beta", "alpha"])
|
||||
self.assertEqual(second["alpha"]["missing_names"], {"a.bin"})
|
||||
|
||||
def test_a_changed_input_computes_again(self):
|
||||
self._run()
|
||||
for change in (
|
||||
lambda: (self.platforms / "alpha.yml").write_text("platform: Alpha 2\n"),
|
||||
lambda: self.db.write_text('{"files": {}}'),
|
||||
lambda: (self.emulators / "core.yml").write_text("emulator: core\n"),
|
||||
):
|
||||
self.calls.clear()
|
||||
change()
|
||||
self._run()
|
||||
self.assertEqual(self.calls, ["alpha", "beta"])
|
||||
|
||||
def test_another_platform_list_computes_again(self):
|
||||
self._run()
|
||||
self.calls.clear()
|
||||
self._run(("alpha",))
|
||||
self.assertEqual(self.calls, ["alpha"])
|
||||
|
||||
def test_a_platform_without_a_file_is_left_out(self):
|
||||
self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"])
|
||||
self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"])
|
||||
|
||||
def test_one_cache_file_is_kept(self):
|
||||
self._run()
|
||||
(self.platforms / "alpha.yml").write_text("platform: Alpha 3\n")
|
||||
self._run()
|
||||
self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -13,7 +13,14 @@ from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import verify # noqa: E402
|
||||
from exporter.candidates import BIOS, GAME_DATA, UNREAD, EntryCategories # noqa: E402
|
||||
from exporter.candidates import ( # noqa: E402
|
||||
BIOS,
|
||||
GAME_DATA,
|
||||
UNREAD,
|
||||
EntryCategories,
|
||||
ProofIndex,
|
||||
ReadEvidence,
|
||||
)
|
||||
|
||||
PROFILES = {
|
||||
"bluemsx": {
|
||||
@@ -144,5 +151,49 @@ class StrictCountsFirmwareOnly(unittest.TestCase):
|
||||
self.assertEqual(evidence["diabdat.mpq"]["read_by"][0]["emulator"], "devilutionx")
|
||||
|
||||
|
||||
class IndexedProof(unittest.TestCase):
|
||||
"""ProofIndex answers what ReadEvidence.proof answers, reader by reader."""
|
||||
|
||||
def test_every_kind_matches_the_direct_proof(self):
|
||||
md5 = "490f666e1afb15b7362b406ed1cea246"
|
||||
readers = []
|
||||
for profile in (
|
||||
{"files": [{"name": "a.bin", "path": "sys/dir/a.bin"}]},
|
||||
{"files": [{"name": "other.rom", "aliases": ["B.ROM"]}]},
|
||||
{"files": [{"name": "c.bin", "md5": md5}]},
|
||||
{"files": [{"name": "tree/", "type": "directory", "path": "data/tree/"}]},
|
||||
{"files": [], "data_directories": [{"ref": "x", "destination": "assets/sys"}]},
|
||||
{"files": [{"name": "d.bin", "sha1": "ab" * 20, "crc32": "12345678"}]},
|
||||
):
|
||||
evidence = ReadEvidence()
|
||||
evidence.add_profile(profile, {})
|
||||
readers.append(evidence)
|
||||
launcher = ReadEvidence()
|
||||
launcher.add_launchers([{"names": ["scph101.bin"]}])
|
||||
readers.append(launcher)
|
||||
index = ProofIndex(readers)
|
||||
cases = [
|
||||
({"name": "a.bin"}, "bios/sys/dir/a.bin"),
|
||||
({"name": "a.bin"}, "elsewhere/a.bin"),
|
||||
({"name": "x", "aliases": ["b.rom"]}, "x"),
|
||||
({"name": "y", "md5": md5[:29]}, "y"),
|
||||
({"name": "y", "md5": md5.upper()}, "y"),
|
||||
({"name": "z", "md5": md5[:28]}, "z"),
|
||||
({"name": "z"}, "root/data/tree/deep/z"),
|
||||
({"name": "z"}, "data/tree"),
|
||||
({"name": "z"}, "a/assets/sys/z.png"),
|
||||
({"name": "scph101.bin"}, "bios/scph101.bin"),
|
||||
({"name": "q", "crc32": "12345678, 9"}, "q"),
|
||||
({"name": "q"}, ""),
|
||||
]
|
||||
for entry, destination in cases:
|
||||
expected = {
|
||||
position: proof for position, evidence in enumerate(readers)
|
||||
if (proof := evidence.proof(entry, destination)) is not None
|
||||
}
|
||||
with self.subTest(entry=entry, destination=destination):
|
||||
self.assertEqual(index.proofs(entry, destination), expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user