feat: publish the corrected platform files

export_native now writes into exports/ by default, and beside each
platform's corrected file a report naming every change, where the run
printed five per kind. A new site page lists each platform with its
original file, a download of the corrected one and every corrected,
filled or withheld value. Each platform page links to its row.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 09:20:02 +02:00
1 parent 71b59eb2a3
commit ca13e6036d
8 files changed
+317 -7

No files matched your search

+26 -3
View File
@@ -8,7 +8,7 @@ formats that carry code are patched rather than regenerated, so the checker
a platform ships keeps working.
Usage:
python scripts/export_native.py --all --fetch
python scripts/export_native.py --all --fetch # into exports/
python scripts/export_native.py --platform recalbox --upstream-dir up/
python scripts/export_native.py --platform retrobat --refresh-cache
"""
@@ -28,7 +28,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from artifacts import file_lock, write_bytes_atomic, write_text_atomic
from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_text_atomic
import yaml
from common import (
@@ -562,9 +562,32 @@ def export_platform(
if len(issues) > 10:
messages.append(f"and {len(issues) - 10} more validation failures")
write_report(output_dir / f"{platform}.json", {
"platform": platform,
"originals": wanted_sources(exporter, systems, scraped),
"files": sorted(produced),
"summary": summary,
"hashes_corrected": applied,
"hashes_filled": filled,
"sizes_corrected": resized,
"sizes_removed": unsized,
"hashes_cleared": cleared,
"requirements_corrected": requirements,
"withheld": list(withheld),
"invalid": issues,
})
return not issues, messages
def write_report(path: Path, report: dict) -> None:
"""What one platform's export changed, for the page that publishes it.
The run prints five examples per kind of change. The page lists them
all, so the maintainer reading it finds each line of the diff named.
"""
write_if_changed(str(path), json.dumps(report, indent=2) + "\n")
def run(
platforms: list[str],
truth_dir: str,
@@ -626,7 +649,7 @@ def main() -> None:
group.add_argument("--all", action="store_true", help="every platform")
group.add_argument("--platform", type=platform_arg, help="a single platform")
parser.add_argument(
"--output-dir", default="dist/upstream",
"--output-dir", default="exports",
help="directory the corrected platform files go to (default: %(default)s)",
)
parser.add_argument(
+21
View File
@@ -64,6 +64,7 @@ from generate_readme import (
release_totals,
)
from provenance_report import build_report, distinct_missing
from siteexports import generate_exports_page, load_export_reports
import release_record
DOCS_DIR = "docs"
@@ -1250,6 +1251,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
"data.md": "Versioned RetroBIOS JSON API, CSV exports, SQLite snapshot, schemas, checksums and data interpretation guidance.",
"cross-reference.md": "Cross-reference from retrogaming platforms to emulator cores, systems, upstream projects and profiled firmware files.",
"gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.",
"exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) rewritten with the hashes and files the emulator source code proves.",
"provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.",
"which-pack.md": "One-line automatic RetroBIOS installation and platform-specific pack destinations, with verification and release caveats.",
}
@@ -1800,6 +1802,10 @@ def _render_platform_systems(
return lines
# Platforms with a corrected file published by export_native.py.
_EXPORTED: set[str] = set()
def generate_platform_page(
name: str,
cov: dict,
@@ -1891,6 +1897,8 @@ def generate_platform_page(
)
note = " (a branch: the revision read is not recorded)" if moving else ""
lines.append(f"| Transcribed from | <{transcribed}>{note} |")
if name in _EXPORTED:
lines.append(f"| Corrected file | [Download it and see what changed](../exports.md#{name}) |")
contrib_list = (registry or {}).get(name, {}).get("contributed_by", [])
if contrib_list:
for cb in contrib_list:
@@ -4639,6 +4647,7 @@ def generate_mkdocs_nav(
{"Cross-reference": "cross-reference.md"},
{"Gap analysis": "gaps.md"},
{"Dump provenance": "provenance.md"},
{"Corrected platform files": "exports.md"},
{"Data & API": "data.md"},
{"Wiki": wiki_nav},
{"Contributing": "contributing.md"},
@@ -4791,6 +4800,9 @@ def _generate(args: argparse.Namespace) -> None:
_build_system_page_map_from_data(manufacturers, coverages, db, profiles)
print(f" {len(_system_page_map)} system IDs mapped to pages")
export_reports = load_export_reports()
_EXPORTED.update(report["platform"] for report in export_reports)
print("Generating platform pages...")
write_if_changed(
str(docs / "platforms" / "index.md"),
@@ -4876,6 +4888,15 @@ def _generate(args: argparse.Namespace) -> None:
),
)
print("Generating corrected platform files page...")
write_if_changed(
str(docs / "exports.md"),
generate_exports_page(
export_reports,
{name: cov["platform"] for name, cov in coverages.items()},
),
)
print("Generating dump provenance page...")
provenance_report = build_report(db, load_provenance_snapshots())
write_if_changed(
+150
View File
@@ -0,0 +1,150 @@
"""The site page that publishes each platform's corrected BIOS file.
export_native.py writes the platform's own file, corrected, under
exports/<platform>/ and what it changed beside it as exports/<platform>.json.
This page reads those reports: the reader finds their platform, the file its
maintainers keep, the corrected copy, and every change named.
"""
from __future__ import annotations
import json
from pathlib import Path
EXPORTS_DIR = "exports"
REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main"
REPO_TREE = "https://github.com/Abdess/retrobios/tree/main"
REPO_RAW = "https://raw.githubusercontent.com/Abdess/retrobios/main"
# Report key, then how a line of it reads on the page.
CHANGE_KINDS = (
("hashes_corrected", "Hashes corrected"),
("hashes_filled", "Hashes filled where the platform had none"),
("sizes_corrected", "Sizes corrected"),
("sizes_removed", "Sizes removed"),
("hashes_cleared", "Hashes cleared"),
("withheld", "Corrections withheld"),
)
def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]:
"""Every platform report the export wrote, by platform name."""
root = Path(directory)
if not root.is_dir():
return []
return [
json.loads(path.read_text(encoding="utf-8"))
for path in sorted(root.glob("*.json"))
]
def _display(platform: str, names: dict[str, str]) -> str:
return names.get(platform) or platform
def _corrected_link(report: dict) -> str:
platform, files = report["platform"], report["files"]
if len(files) == 1:
path = f"{EXPORTS_DIR}/{platform}/{files[0]}"
return f"[`{Path(files[0]).name}`]({REPO_BLOB}/{path})"
return f"[{len(files)} files]({REPO_TREE}/{EXPORTS_DIR}/{platform})"
def _original_link(report: dict) -> str:
originals = report.get("originals") or {}
if len(originals) == 1:
(name, url), = originals.items()
return f"[`{Path(name).name}`]({url})"
return f"{len(originals)} files"
def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
"""The page, names mapping each platform key to the name it displays."""
lines = [
"# Corrected platform files",
"",
"Each platform keeps its own list of the BIOS files it checks: "
"libretro's `System.dat`, Batocera's `batocera-systems`, Recalbox's "
"`es_bios.xml` and so on. Those lists are written by hand and drift "
"from what the emulators actually read. Here is each platform's own "
"file again, with what the emulator source code proves applied. A "
"wrong hash is corrected, a file the platform's emulators need and "
"the list lacks is added, and nothing the platform declares is ever "
"removed. Code the file carries (a checker script, a C# database, a "
"shell library) is patched in place and keeps working.",
"",
"Each file is taken at the revision our data was transcribed from, "
"so a maintainer can diff it against theirs and take the changes "
"line by line.",
"",
]
if not reports:
lines.extend([
"No corrected file has been published yet. They are written by "
"`python scripts/export_native.py --all --fetch`.",
"",
])
return "\n".join(lines)
lines.extend([
"| Platform | Their file | Corrected file | What changed |",
"|----------|------------|----------------|--------------|",
])
for report in reports:
platform = report["platform"]
lines.append(
f"| [{_display(platform, names)}](#{platform}) | "
f"{_original_link(report)} | {_corrected_link(report)} | "
f"{report['summary']} |"
)
lines.append("")
for report in reports:
platform = report["platform"]
lines.extend([f"## {_display(platform, names)} {{ #{platform} }}", ""])
lines.extend([report["summary"][:1].upper() + report["summary"][1:] + ".", ""])
if report.get("invalid"):
lines.extend([
'!!! warning "The corrected file fails the platform\'s own checks"',
"",
*[f" - {issue}" for issue in report["invalid"]],
"",
])
originals = report.get("originals") or {}
files = report["files"]
raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files]
if len(files) == 1:
lines.extend([
f"[Download the corrected file]({raw[0]}){{ .md-button .md-button--primary }} "
f"[See it on GitHub]({REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}){{ .md-button }}",
"",
])
else:
lines.extend([
f"[Browse the {len(files)} corrected files]"
f"({REPO_TREE}/{EXPORTS_DIR}/{platform}){{ .md-button .md-button--primary }}",
"",
])
if len(originals) > 1:
lines.extend([
f'??? note "The {len(originals)} files this export patches"',
"",
*[f" - [`{name}`]({url})" for name, url in sorted(originals.items())],
"",
])
elif originals:
(name, url), = originals.items()
lines.extend([f"Patched from [`{name}`]({url}).", ""])
for key, title in CHANGE_KINDS:
items = report.get(key) or []
if not items:
continue
lines.extend([
f'??? note "{title} ({len(items)})"',
"",
*[f" - {item}" if key == "withheld" else f" - `{item}`" for item in items],
"",
])
return "\n".join(lines)