feat: publish the corrected platform files

export_native now writes into exports/ by default, and beside each
platform's corrected file a report naming every change, where the run
printed five per kind. A new site page lists each platform with its
original file, a download of the corrected one and every corrected,
filled or withheld value. Each platform page links to its row.

.github/workflows/deploy-site.yml

@@ -8,6 +8,7 @@ on:
       - "emulators/**"
       - "provenance/**"
       - "wiki/**"
+      - "exports/**"
       # Every script the build runs or imports, and no other
       # (tests/test_workflow_paths.py).
       - "scripts/artifacts.py"
@@ -34,6 +35,7 @@ on:
       - "scripts/restore_large_files.py"
       - "scripts/safeparse.py"
       - "scripts/sect233r1.py"
+      - "scripts/siteexports.py"
       - "scripts/siterender.py"
       - "scripts/sitesources.py"
       - "scripts/sitewrite.py"

scripts/export_native.py

@@ -8,7 +8,7 @@ formats that carry code are patched rather than regenerated, so the checker
 a platform ships keeps working.
 
 Usage:
-    python scripts/export_native.py --all --fetch
+    python scripts/export_native.py --all --fetch           # into exports/
     python scripts/export_native.py --platform recalbox --upstream-dir up/
     python scripts/export_native.py --platform retrobat --refresh-cache
 """
@@ -28,7 +28,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
-from artifacts import file_lock, write_bytes_atomic, write_text_atomic
+from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_text_atomic
 import yaml
 
 from common import (
@@ -562,9 +562,32 @@ def export_platform(
     if len(issues) > 10:
         messages.append(f"and {len(issues) - 10} more validation failures")
 
+    write_report(output_dir / f"{platform}.json", {
+        "platform": platform,
+        "originals": wanted_sources(exporter, systems, scraped),
+        "files": sorted(produced),
+        "summary": summary,
+        "hashes_corrected": applied,
+        "hashes_filled": filled,
+        "sizes_corrected": resized,
+        "sizes_removed": unsized,
+        "hashes_cleared": cleared,
+        "requirements_corrected": requirements,
+        "withheld": list(withheld),
+        "invalid": issues,
+    })
     return not issues, messages
 
 
+def write_report(path: Path, report: dict) -> None:
+    """What one platform's export changed, for the page that publishes it.
+
+    The run prints five examples per kind of change. The page lists them
+    all, so the maintainer reading it finds each line of the diff named.
+    """
+    write_if_changed(str(path), json.dumps(report, indent=2) + "\n")
+
+
 def run(
     platforms: list[str],
     truth_dir: str,
@@ -626,7 +649,7 @@ def main() -> None:
     group.add_argument("--all", action="store_true", help="every platform")
     group.add_argument("--platform", type=platform_arg, help="a single platform")
     parser.add_argument(
-        "--output-dir", default="dist/upstream",
+        "--output-dir", default="exports",
         help="directory the corrected platform files go to (default: %(default)s)",
     )
     parser.add_argument(

scripts/generate_site.py

@@ -64,6 +64,7 @@ from generate_readme import (
     release_totals,
 )
 from provenance_report import build_report, distinct_missing
+from siteexports import generate_exports_page, load_export_reports
 import release_record
 
 DOCS_DIR = "docs"
@@ -1250,6 +1251,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
         "data.md": "Versioned RetroBIOS JSON API, CSV exports, SQLite snapshot, schemas, checksums and data interpretation guidance.",
         "cross-reference.md": "Cross-reference from retrogaming platforms to emulator cores, systems, upstream projects and profiled firmware files.",
         "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.",
+        "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) rewritten with the hashes and files the emulator source code proves.",
         "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.",
         "which-pack.md": "One-line automatic RetroBIOS installation and platform-specific pack destinations, with verification and release caveats.",
     }
@@ -1800,6 +1802,10 @@ def _render_platform_systems(
     return lines
 
 
+# Platforms with a corrected file published by export_native.py.
+_EXPORTED: set[str] = set()
+
+
 def generate_platform_page(
     name: str,
     cov: dict,
@@ -1891,6 +1897,8 @@ def generate_platform_page(
         )
         note = " (a branch: the revision read is not recorded)" if moving else ""
         lines.append(f"| Transcribed from | <{transcribed}>{note} |")
+    if name in _EXPORTED:
+        lines.append(f"| Corrected file | [Download it and see what changed](../exports.md#{name}) |")
     contrib_list = (registry or {}).get(name, {}).get("contributed_by", [])
     if contrib_list:
         for cb in contrib_list:
@@ -4639,6 +4647,7 @@ def generate_mkdocs_nav(
         {"Cross-reference": "cross-reference.md"},
         {"Gap analysis": "gaps.md"},
         {"Dump provenance": "provenance.md"},
+        {"Corrected platform files": "exports.md"},
         {"Data & API": "data.md"},
         {"Wiki": wiki_nav},
         {"Contributing": "contributing.md"},
@@ -4791,6 +4800,9 @@ def _generate(args: argparse.Namespace) -> None:
     _build_system_page_map_from_data(manufacturers, coverages, db, profiles)
     print(f"  {len(_system_page_map)} system IDs mapped to pages")
 
+    export_reports = load_export_reports()
+    _EXPORTED.update(report["platform"] for report in export_reports)
+
     print("Generating platform pages...")
     write_if_changed(
         str(docs / "platforms" / "index.md"),
@@ -4876,6 +4888,15 @@ def _generate(args: argparse.Namespace) -> None:
         ),
     )
 
+    print("Generating corrected platform files page...")
+    write_if_changed(
+        str(docs / "exports.md"),
+        generate_exports_page(
+            export_reports,
+            {name: cov["platform"] for name, cov in coverages.items()},
+        ),
+    )
+
     print("Generating dump provenance page...")
     provenance_report = build_report(db, load_provenance_snapshots())
     write_if_changed(

scripts/siteexports.py

@@ -0,0 +1,150 @@
+"""The site page that publishes each platform's corrected BIOS file.
+
+export_native.py writes the platform's own file, corrected, under
+exports/<platform>/ and what it changed beside it as exports/<platform>.json.
+This page reads those reports: the reader finds their platform, the file its
+maintainers keep, the corrected copy, and every change named.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+EXPORTS_DIR = "exports"
+REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main"
+REPO_TREE = "https://github.com/Abdess/retrobios/tree/main"
+REPO_RAW = "https://raw.githubusercontent.com/Abdess/retrobios/main"
+
+# Report key, then how a line of it reads on the page.
+CHANGE_KINDS = (
+    ("hashes_corrected", "Hashes corrected"),
+    ("hashes_filled", "Hashes filled where the platform had none"),
+    ("sizes_corrected", "Sizes corrected"),
+    ("sizes_removed", "Sizes removed"),
+    ("hashes_cleared", "Hashes cleared"),
+    ("withheld", "Corrections withheld"),
+)
+
+
+def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]:
+    """Every platform report the export wrote, by platform name."""
+    root = Path(directory)
+    if not root.is_dir():
+        return []
+    return [
+        json.loads(path.read_text(encoding="utf-8"))
+        for path in sorted(root.glob("*.json"))
+    ]
+
+
+def _display(platform: str, names: dict[str, str]) -> str:
+    return names.get(platform) or platform
+
+
+def _corrected_link(report: dict) -> str:
+    platform, files = report["platform"], report["files"]
+    if len(files) == 1:
+        path = f"{EXPORTS_DIR}/{platform}/{files[0]}"
+        return f"[`{Path(files[0]).name}`]({REPO_BLOB}/{path})"
+    return f"[{len(files)} files]({REPO_TREE}/{EXPORTS_DIR}/{platform})"
+
+
+def _original_link(report: dict) -> str:
+    originals = report.get("originals") or {}
+    if len(originals) == 1:
+        (name, url), = originals.items()
+        return f"[`{Path(name).name}`]({url})"
+    return f"{len(originals)} files"
+
+
+def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
+    """The page, names mapping each platform key to the name it displays."""
+    lines = [
+        "# Corrected platform files",
+        "",
+        "Each platform keeps its own list of the BIOS files it checks: "
+        "libretro's `System.dat`, Batocera's `batocera-systems`, Recalbox's "
+        "`es_bios.xml` and so on. Those lists are written by hand and drift "
+        "from what the emulators actually read. Here is each platform's own "
+        "file again, with what the emulator source code proves applied. A "
+        "wrong hash is corrected, a file the platform's emulators need and "
+        "the list lacks is added, and nothing the platform declares is ever "
+        "removed. Code the file carries (a checker script, a C# database, a "
+        "shell library) is patched in place and keeps working.",
+        "",
+        "Each file is taken at the revision our data was transcribed from, "
+        "so a maintainer can diff it against theirs and take the changes "
+        "line by line.",
+        "",
+    ]
+    if not reports:
+        lines.extend([
+            "No corrected file has been published yet. They are written by "
+            "`python scripts/export_native.py --all --fetch`.",
+            "",
+        ])
+        return "\n".join(lines)
+
+    lines.extend([
+        "| Platform | Their file | Corrected file | What changed |",
+        "|----------|------------|----------------|--------------|",
+    ])
+    for report in reports:
+        platform = report["platform"]
+        lines.append(
+            f"| [{_display(platform, names)}](#{platform}) | "
+            f"{_original_link(report)} | {_corrected_link(report)} | "
+            f"{report['summary']} |"
+        )
+    lines.append("")
+
+    for report in reports:
+        platform = report["platform"]
+        lines.extend([f"## {_display(platform, names)} {{ #{platform} }}", ""])
+        lines.extend([report["summary"][:1].upper() + report["summary"][1:] + ".", ""])
+        if report.get("invalid"):
+            lines.extend([
+                '!!! warning "The corrected file fails the platform\'s own checks"',
+                "",
+                *[f"    - {issue}" for issue in report["invalid"]],
+                "",
+            ])
+
+        originals = report.get("originals") or {}
+        files = report["files"]
+        raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files]
+        if len(files) == 1:
+            lines.extend([
+                f"[Download the corrected file]({raw[0]}){{ .md-button .md-button--primary }} "
+                f"[See it on GitHub]({REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}){{ .md-button }}",
+                "",
+            ])
+        else:
+            lines.extend([
+                f"[Browse the {len(files)} corrected files]"
+                f"({REPO_TREE}/{EXPORTS_DIR}/{platform}){{ .md-button .md-button--primary }}",
+                "",
+            ])
+        if len(originals) > 1:
+            lines.extend([
+                f'??? note "The {len(originals)} files this export patches"',
+                "",
+                *[f"    - [`{name}`]({url})" for name, url in sorted(originals.items())],
+                "",
+            ])
+        elif originals:
+            (name, url), = originals.items()
+            lines.extend([f"Patched from [`{name}`]({url}).", ""])
+
+        for key, title in CHANGE_KINDS:
+            items = report.get(key) or []
+            if not items:
+                continue
+            lines.extend([
+                f'??? note "{title} ({len(items)})"',
+                "",
+                *[f"    - {item}" if key == "withheld" else f"    - `{item}`" for item in items],
+                "",
+            ])
+    return "\n".join(lines)

tests/test_export_report.py

@@ -0,0 +1,108 @@
+"""The corrected platform files are published with every change named.
+
+export_native printed five examples per kind of change and nothing else
+recorded the rest: a platform maintainer had no page to fetch the corrected
+file from and no list to check the diff against.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+
+import export_native  # noqa: E402
+from siteexports import generate_exports_page, load_export_reports  # noqa: E402
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def _report(platform: str, files: list[str], **changes: object) -> dict:
+    report = {
+        "platform": platform,
+        "originals": {f: f"https://example.org/{platform}/{f}" for f in files},
+        "files": files,
+        "summary": "3 kept, 1 added, 7 hashes corrected, 0 requirements corrected",
+        "hashes_corrected": [], "hashes_filled": [], "sizes_corrected": [],
+        "sizes_removed": [], "hashes_cleared": [], "requirements_corrected": 0,
+        "withheld": [], "invalid": [],
+    }
+    report.update(changes)
+    return report
+
+
+class ExportReportIsWrittenAndRead(unittest.TestCase):
+    def test_report_round_trips_through_the_page_loader(self):
+        with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch:
+            report = _report("recalbox", ["es_bios.xml"])
+            export_native.write_report(Path(scratch) / "recalbox.json", report)
+            (Path(scratch) / "recalbox").mkdir()
+            self.assertEqual(load_export_reports(scratch), [report])
+
+    def test_export_keeps_every_change_not_the_printed_five(self):
+        source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
+        body = source[source.index("def export_platform("):source.index("def write_report(")]
+        call = body[body.index("write_report("):]
+        for key, value in (
+            ("hashes_corrected", "applied"), ("hashes_filled", "filled"),
+            ("sizes_corrected", "resized"), ("sizes_removed", "unsized"),
+            ("hashes_cleared", "cleared"), ("invalid", "issues"),
+        ):
+            self.assertIn(f'"{key}": {value},', call)
+
+    def test_default_output_is_the_published_directory(self):
+        source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
+        self.assertIn('"--output-dir", default="exports"', source)
+
+
+class CorrectedFilesPage(unittest.TestCase):
+    def setUp(self):
+        corrected = [f"sys/file{i}.bin md5" for i in range(12)]
+        self.reports = [
+            _report("recalbox", ["es_bios.xml"], hashes_corrected=corrected),
+            _report("retropie", ["scriptmodules/a.sh", "scriptmodules/b.sh"],
+                    invalid=["bash -n failed on a.sh"],
+                    withheld=["a size the format has no field for"]),
+        ]
+        self.page = generate_exports_page(self.reports, {"recalbox": "Recalbox"})
+
+    def test_every_platform_has_a_row_and_a_section(self):
+        self.assertIn("| [Recalbox](#recalbox) | [`es_bios.xml`]", self.page)
+        self.assertIn("## Recalbox { #recalbox }", self.page)
+        self.assertIn("## retropie { #retropie }", self.page)
+
+    def test_single_file_is_downloadable_raw(self):
+        self.assertIn(
+            "(https://raw.githubusercontent.com/Abdess/retrobios/main/exports/recalbox/es_bios.xml)",
+            self.page,
+        )
+
+    def test_several_files_link_to_their_directory(self):
+        self.assertIn("tree/main/exports/retropie", self.page)
+        self.assertIn('??? note "The 2 files this export patches"', self.page)
+
+    def test_every_change_is_listed(self):
+        self.assertIn('??? note "Hashes corrected (12)"', self.page)
+        for i in range(12):
+            self.assertIn(f"    - `sys/file{i}.bin md5`", self.page)
+
+    def test_a_failing_file_is_said_to_fail(self):
+        self.assertIn("    - bash -n failed on a.sh", self.page)
+        self.assertIn("    - a size the format has no field for", self.page)
+
+    def test_nothing_published_says_how_to_build(self):
+        self.assertIn("export_native.py --all --fetch", generate_exports_page([], {}))
+
+    def test_site_has_the_page_in_its_nav(self):
+        source = (ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
+        self.assertIn('{"Corrected platform files": "exports.md"}', source)
+        workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8")
+        self.assertIn('"exports/**"', workflow)
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/advanced-usage.md

@@ -339,10 +339,16 @@ the truth knows and the platform lacks are added.
 ```bash
 python scripts/export_native.py --platform batocera --fetch   # batocera-systems
 python scripts/export_native.py --platform recalbox --fetch   # es_bios.xml
-python scripts/export_native.py --all --fetch --output-dir dist/upstream/
+python scripts/export_native.py --all --fetch                # every platform, into exports/
 ```
 
-Seven of the formats carry code as well as data - Batocera's and ROCKNIX's
+The corrected files are committed under `exports/<platform>/`, and
+`exports/<platform>.json` names every change the run made. The site's
+[corrected platform files](../exports.md) page is built from those reports:
+one row per platform, a download link, and each corrected hash, filled hash
+and withheld correction listed in full.
+
+Seven of the formats carry code as well as data: Batocera's and ROCKNIX's
 scripts, EmuDeck's shell library, BizHawk's C# database, RetroDECK's
 component manifests, MiSTer's BiosDB, RetroPie's scriptmodules. Those are
 patched from the platform's own file, which `--fetch` downloads once into

wiki/testing-guide.md

@@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards.
 | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
 | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
 | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
-| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_exporters`, `test_truth_diff`, `test_truth_layout` |
+| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
 | Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_hashes`, `test_workflow_paths` |
 

wiki/tools.md

@@ -244,7 +244,7 @@ archived included.
 
 ```bash
 python scripts/export_native.py --platform batocera --fetch
-python scripts/export_native.py --all --fetch --output-dir dist/upstream/
+python scripts/export_native.py --all --fetch   # into exports/, published on the site
 ```
 
 `--fetch` downloads the platform's own file into `.cache/upstream-native/`,