diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 45ec7700..e7685191 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -8,6 +8,7 @@ on: - "emulators/**" - "provenance/**" - "wiki/**" + - "exports/**" # Every script the build runs or imports, and no other # (tests/test_workflow_paths.py). - "scripts/artifacts.py" @@ -34,6 +35,7 @@ on: - "scripts/restore_large_files.py" - "scripts/safeparse.py" - "scripts/sect233r1.py" + - "scripts/siteexports.py" - "scripts/siterender.py" - "scripts/sitesources.py" - "scripts/sitewrite.py" diff --git a/scripts/export_native.py b/scripts/export_native.py index 2465b119..cca650d1 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -8,7 +8,7 @@ formats that carry code are patched rather than regenerated, so the checker a platform ships keeps working. Usage: - python scripts/export_native.py --all --fetch + python scripts/export_native.py --all --fetch # into exports/ python scripts/export_native.py --platform recalbox --upstream-dir up/ python scripts/export_native.py --platform retrobat --refresh-cache """ @@ -28,7 +28,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from artifacts import file_lock, write_bytes_atomic, write_text_atomic +from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_text_atomic import yaml from common import ( @@ -562,9 +562,32 @@ def export_platform( if len(issues) > 10: messages.append(f"and {len(issues) - 10} more validation failures") + write_report(output_dir / f"{platform}.json", { + "platform": platform, + "originals": wanted_sources(exporter, systems, scraped), + "files": sorted(produced), + "summary": summary, + "hashes_corrected": applied, + "hashes_filled": filled, + "sizes_corrected": resized, + "sizes_removed": unsized, + "hashes_cleared": cleared, + "requirements_corrected": requirements, + "withheld": list(withheld), + "invalid": issues, + }) return not issues, messages +def write_report(path: Path, report: dict) -> None: + """What one platform's export changed, for the page that publishes it. + + The run prints five examples per kind of change. The page lists them + all, so the maintainer reading it finds each line of the diff named. + """ + write_if_changed(str(path), json.dumps(report, indent=2) + "\n") + + def run( platforms: list[str], truth_dir: str, @@ -626,7 +649,7 @@ def main() -> None: group.add_argument("--all", action="store_true", help="every platform") group.add_argument("--platform", type=platform_arg, help="a single platform") parser.add_argument( - "--output-dir", default="dist/upstream", + "--output-dir", default="exports", help="directory the corrected platform files go to (default: %(default)s)", ) parser.add_argument( diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 574c9110..4293dc54 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -64,6 +64,7 @@ from generate_readme import ( release_totals, ) from provenance_report import build_report, distinct_missing +from siteexports import generate_exports_page, load_export_reports import release_record DOCS_DIR = "docs" @@ -1250,6 +1251,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: "data.md": "Versioned RetroBIOS JSON API, CSV exports, SQLite snapshot, schemas, checksums and data interpretation guidance.", "cross-reference.md": "Cross-reference from retrogaming platforms to emulator cores, systems, upstream projects and profiled firmware files.", "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.", + "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) rewritten with the hashes and files the emulator source code proves.", "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.", "which-pack.md": "One-line automatic RetroBIOS installation and platform-specific pack destinations, with verification and release caveats.", } @@ -1800,6 +1802,10 @@ def _render_platform_systems( return lines +# Platforms with a corrected file published by export_native.py. +_EXPORTED: set[str] = set() + + def generate_platform_page( name: str, cov: dict, @@ -1891,6 +1897,8 @@ def generate_platform_page( ) note = " (a branch: the revision read is not recorded)" if moving else "" lines.append(f"| Transcribed from | <{transcribed}>{note} |") + if name in _EXPORTED: + lines.append(f"| Corrected file | [Download it and see what changed](../exports.md#{name}) |") contrib_list = (registry or {}).get(name, {}).get("contributed_by", []) if contrib_list: for cb in contrib_list: @@ -4639,6 +4647,7 @@ def generate_mkdocs_nav( {"Cross-reference": "cross-reference.md"}, {"Gap analysis": "gaps.md"}, {"Dump provenance": "provenance.md"}, + {"Corrected platform files": "exports.md"}, {"Data & API": "data.md"}, {"Wiki": wiki_nav}, {"Contributing": "contributing.md"}, @@ -4791,6 +4800,9 @@ def _generate(args: argparse.Namespace) -> None: _build_system_page_map_from_data(manufacturers, coverages, db, profiles) print(f" {len(_system_page_map)} system IDs mapped to pages") + export_reports = load_export_reports() + _EXPORTED.update(report["platform"] for report in export_reports) + print("Generating platform pages...") write_if_changed( str(docs / "platforms" / "index.md"), @@ -4876,6 +4888,15 @@ def _generate(args: argparse.Namespace) -> None: ), ) + print("Generating corrected platform files page...") + write_if_changed( + str(docs / "exports.md"), + generate_exports_page( + export_reports, + {name: cov["platform"] for name, cov in coverages.items()}, + ), + ) + print("Generating dump provenance page...") provenance_report = build_report(db, load_provenance_snapshots()) write_if_changed( diff --git a/scripts/siteexports.py b/scripts/siteexports.py new file mode 100644 index 00000000..d22a17db --- /dev/null +++ b/scripts/siteexports.py @@ -0,0 +1,150 @@ +"""The site page that publishes each platform's corrected BIOS file. + +export_native.py writes the platform's own file, corrected, under +exports// and what it changed beside it as exports/.json. +This page reads those reports: the reader finds their platform, the file its +maintainers keep, the corrected copy, and every change named. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +EXPORTS_DIR = "exports" +REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main" +REPO_TREE = "https://github.com/Abdess/retrobios/tree/main" +REPO_RAW = "https://raw.githubusercontent.com/Abdess/retrobios/main" + +# Report key, then how a line of it reads on the page. +CHANGE_KINDS = ( + ("hashes_corrected", "Hashes corrected"), + ("hashes_filled", "Hashes filled where the platform had none"), + ("sizes_corrected", "Sizes corrected"), + ("sizes_removed", "Sizes removed"), + ("hashes_cleared", "Hashes cleared"), + ("withheld", "Corrections withheld"), +) + + +def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]: + """Every platform report the export wrote, by platform name.""" + root = Path(directory) + if not root.is_dir(): + return [] + return [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(root.glob("*.json")) + ] + + +def _display(platform: str, names: dict[str, str]) -> str: + return names.get(platform) or platform + + +def _corrected_link(report: dict) -> str: + platform, files = report["platform"], report["files"] + if len(files) == 1: + path = f"{EXPORTS_DIR}/{platform}/{files[0]}" + return f"[`{Path(files[0]).name}`]({REPO_BLOB}/{path})" + return f"[{len(files)} files]({REPO_TREE}/{EXPORTS_DIR}/{platform})" + + +def _original_link(report: dict) -> str: + originals = report.get("originals") or {} + if len(originals) == 1: + (name, url), = originals.items() + return f"[`{Path(name).name}`]({url})" + return f"{len(originals)} files" + + +def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: + """The page, names mapping each platform key to the name it displays.""" + lines = [ + "# Corrected platform files", + "", + "Each platform keeps its own list of the BIOS files it checks: " + "libretro's `System.dat`, Batocera's `batocera-systems`, Recalbox's " + "`es_bios.xml` and so on. Those lists are written by hand and drift " + "from what the emulators actually read. Here is each platform's own " + "file again, with what the emulator source code proves applied. A " + "wrong hash is corrected, a file the platform's emulators need and " + "the list lacks is added, and nothing the platform declares is ever " + "removed. Code the file carries (a checker script, a C# database, a " + "shell library) is patched in place and keeps working.", + "", + "Each file is taken at the revision our data was transcribed from, " + "so a maintainer can diff it against theirs and take the changes " + "line by line.", + "", + ] + if not reports: + lines.extend([ + "No corrected file has been published yet. They are written by " + "`python scripts/export_native.py --all --fetch`.", + "", + ]) + return "\n".join(lines) + + lines.extend([ + "| Platform | Their file | Corrected file | What changed |", + "|----------|------------|----------------|--------------|", + ]) + for report in reports: + platform = report["platform"] + lines.append( + f"| [{_display(platform, names)}](#{platform}) | " + f"{_original_link(report)} | {_corrected_link(report)} | " + f"{report['summary']} |" + ) + lines.append("") + + for report in reports: + platform = report["platform"] + lines.extend([f"## {_display(platform, names)} {{ #{platform} }}", ""]) + lines.extend([report["summary"][:1].upper() + report["summary"][1:] + ".", ""]) + if report.get("invalid"): + lines.extend([ + '!!! warning "The corrected file fails the platform\'s own checks"', + "", + *[f" - {issue}" for issue in report["invalid"]], + "", + ]) + + originals = report.get("originals") or {} + files = report["files"] + raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files] + if len(files) == 1: + lines.extend([ + f"[Download the corrected file]({raw[0]}){{ .md-button .md-button--primary }} " + f"[See it on GitHub]({REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}){{ .md-button }}", + "", + ]) + else: + lines.extend([ + f"[Browse the {len(files)} corrected files]" + f"({REPO_TREE}/{EXPORTS_DIR}/{platform}){{ .md-button .md-button--primary }}", + "", + ]) + if len(originals) > 1: + lines.extend([ + f'??? note "The {len(originals)} files this export patches"', + "", + *[f" - [`{name}`]({url})" for name, url in sorted(originals.items())], + "", + ]) + elif originals: + (name, url), = originals.items() + lines.extend([f"Patched from [`{name}`]({url}).", ""]) + + for key, title in CHANGE_KINDS: + items = report.get(key) or [] + if not items: + continue + lines.extend([ + f'??? note "{title} ({len(items)})"', + "", + *[f" - {item}" if key == "withheld" else f" - `{item}`" for item in items], + "", + ]) + return "\n".join(lines) diff --git a/tests/test_export_report.py b/tests/test_export_report.py new file mode 100644 index 00000000..fb8e6c9a --- /dev/null +++ b/tests/test_export_report.py @@ -0,0 +1,108 @@ +"""The corrected platform files are published with every change named. + +export_native printed five examples per kind of change and nothing else +recorded the rest: a platform maintainer had no page to fetch the corrected +file from and no list to check the diff against. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import export_native # noqa: E402 +from siteexports import generate_exports_page, load_export_reports # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent + + +def _report(platform: str, files: list[str], **changes: object) -> dict: + report = { + "platform": platform, + "originals": {f: f"https://example.org/{platform}/{f}" for f in files}, + "files": files, + "summary": "3 kept, 1 added, 7 hashes corrected, 0 requirements corrected", + "hashes_corrected": [], "hashes_filled": [], "sizes_corrected": [], + "sizes_removed": [], "hashes_cleared": [], "requirements_corrected": 0, + "withheld": [], "invalid": [], + } + report.update(changes) + return report + + +class ExportReportIsWrittenAndRead(unittest.TestCase): + def test_report_round_trips_through_the_page_loader(self): + with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch: + report = _report("recalbox", ["es_bios.xml"]) + export_native.write_report(Path(scratch) / "recalbox.json", report) + (Path(scratch) / "recalbox").mkdir() + self.assertEqual(load_export_reports(scratch), [report]) + + def test_export_keeps_every_change_not_the_printed_five(self): + source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8") + body = source[source.index("def export_platform("):source.index("def write_report(")] + call = body[body.index("write_report("):] + for key, value in ( + ("hashes_corrected", "applied"), ("hashes_filled", "filled"), + ("sizes_corrected", "resized"), ("sizes_removed", "unsized"), + ("hashes_cleared", "cleared"), ("invalid", "issues"), + ): + self.assertIn(f'"{key}": {value},', call) + + def test_default_output_is_the_published_directory(self): + source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8") + self.assertIn('"--output-dir", default="exports"', source) + + +class CorrectedFilesPage(unittest.TestCase): + def setUp(self): + corrected = [f"sys/file{i}.bin md5" for i in range(12)] + self.reports = [ + _report("recalbox", ["es_bios.xml"], hashes_corrected=corrected), + _report("retropie", ["scriptmodules/a.sh", "scriptmodules/b.sh"], + invalid=["bash -n failed on a.sh"], + withheld=["a size the format has no field for"]), + ] + self.page = generate_exports_page(self.reports, {"recalbox": "Recalbox"}) + + def test_every_platform_has_a_row_and_a_section(self): + self.assertIn("| [Recalbox](#recalbox) | [`es_bios.xml`]", self.page) + self.assertIn("## Recalbox { #recalbox }", self.page) + self.assertIn("## retropie { #retropie }", self.page) + + def test_single_file_is_downloadable_raw(self): + self.assertIn( + "(https://raw.githubusercontent.com/Abdess/retrobios/main/exports/recalbox/es_bios.xml)", + self.page, + ) + + def test_several_files_link_to_their_directory(self): + self.assertIn("tree/main/exports/retropie", self.page) + self.assertIn('??? note "The 2 files this export patches"', self.page) + + def test_every_change_is_listed(self): + self.assertIn('??? note "Hashes corrected (12)"', self.page) + for i in range(12): + self.assertIn(f" - `sys/file{i}.bin md5`", self.page) + + def test_a_failing_file_is_said_to_fail(self): + self.assertIn(" - bash -n failed on a.sh", self.page) + self.assertIn(" - a size the format has no field for", self.page) + + def test_nothing_published_says_how_to_build(self): + self.assertIn("export_native.py --all --fetch", generate_exports_page([], {})) + + def test_site_has_the_page_in_its_nav(self): + source = (ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8") + self.assertIn('{"Corrected platform files": "exports.md"}', source) + workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8") + self.assertIn('"exports/**"', workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index 60f3ae7e..02b755f2 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -339,10 +339,16 @@ the truth knows and the platform lacks are added. ```bash python scripts/export_native.py --platform batocera --fetch # batocera-systems python scripts/export_native.py --platform recalbox --fetch # es_bios.xml -python scripts/export_native.py --all --fetch --output-dir dist/upstream/ +python scripts/export_native.py --all --fetch # every platform, into exports/ ``` -Seven of the formats carry code as well as data - Batocera's and ROCKNIX's +The corrected files are committed under `exports//`, and +`exports/.json` names every change the run made. The site's +[corrected platform files](../exports.md) page is built from those reports: +one row per platform, a download link, and each corrected hash, filled hash +and withheld correction listed in full. + +Seven of the formats carry code as well as data: Batocera's and ROCKNIX's scripts, EmuDeck's shell library, BizHawk's C# database, RetroDECK's component manifests, MiSTer's BiosDB, RetroPie's scriptmodules. Those are patched from the platform's own file, which `--fetch` downloads once into diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 5a952c00..206e0208 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards. | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | -| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | +| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | | Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` | diff --git a/wiki/tools.md b/wiki/tools.md index 77c7d2ee..e523bd61 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -244,7 +244,7 @@ archived included. ```bash python scripts/export_native.py --platform batocera --fetch -python scripts/export_native.py --all --fetch --output-dir dist/upstream/ +python scripts/export_native.py --all --fetch # into exports/, published on the site ``` `--fetch` downloads the platform's own file into `.cache/upstream-native/`,