diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index c8f4294d..84a54eb3 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -292,10 +292,13 @@ def generate_readme(db: dict, platforms_dir: str) -> str: "", "Coverage is measured against the file list each platform declares," " using that platform's own verification mode.", - "Source-backed counts the files whose expectations were also read" - " from an emulator's source code; the rest rely on the platform" - " list alone. A dash means no profiled emulator applies to the" - " platform, whose own source is then the only authority.", + "Source-backed counts the files whose content the emulator's own" + " code checks: a size or hash read from its source, reproduced at" + " verification. Files a profile documents without a content check" + " are counted separately on the" + f" [gap analysis]({SITE_URL}gaps/) page. A dash means no profiled" + " emulator applies to the platform, whose own source is then the" + " only authority.", "Where platform lists and emulator source code disagree, the" f" differences are tracked in the [gap analysis]({SITE_URL}gaps/).", "", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 047874a1..5964063e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1522,8 +1522,8 @@ def generate_gap_analysis( lines.extend([ "## Verification by Platform", "", - "| Platform | Files | Verified | Untested | Missing | Mode | Source-backed |", - "|----------|------:|---------:|---------:|--------:|------|--------------:|", + "| Platform | Files | Verified | Untested | Missing | Mode | Profiled | Source-backed |", + "|----------|------:|---------:|---------:|--------:|------|---------:|--------------:|", ]) for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): @@ -1542,12 +1542,14 @@ def generate_gap_analysis( ) gt = cov["ground_truth"] if not gt.get("applicable", True): - gt_cell = "-" + gt_cell = prof_cell = "-" elif gt["total"]: gt_pct = f"{gt['with_validation'] / gt['total'] * 100:.0f}%" gt_cell = f"{gt['with_validation']}/{gt['total']} ({gt_pct})" + prof_pct = f"{gt.get('with_profile', 0) / gt['total'] * 100:.0f}%" + prof_cell = f"{gt.get('with_profile', 0)}/{gt['total']} ({prof_pct})" else: - gt_cell = "0/0" + gt_cell = prof_cell = "0/0" lines.append( f"| [{display}](platforms/{pname}.md) " f"| {cov['total']} " @@ -1555,6 +1557,7 @@ def generate_gap_analysis( f"| {untested_str} " f"| {missing_str} " f"| {cov['mode']} " + f"| {prof_cell} " f"| {gt_cell} |" ) lines.extend([ @@ -1562,10 +1565,13 @@ def generate_gap_analysis( "Verification follows each platform's own runtime check " "([how each mode works](wiki/verification-modes.md)): the counts " "measure the repository against the file list each platform declares. " - "Source-backed counts the files whose expectations were also read " - "from an emulator's source code; the rest rely on the platform list " - "alone. A dash means no profiled emulator applies to the platform, " - "whose own source is then the only authority.", + "Profiled counts the files documented in an emulator profile (name or " + "alias read from source). Source-backed is stricter: the emulator's " + "own code checks the file's content (a size or hash read from its " + "source, reproduced at verification). A file can be profiled without " + "being source-backed when the code loads it but never checks it. " + "A dash means no profiled emulator applies to the platform, whose " + "own source is then the only authority.", "", ]) diff --git a/scripts/verify.py b/scripts/verify.py index 2476ca68..1fd53079 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -835,6 +835,15 @@ def verify_platform( # Ground truth coverage gt_filenames = set(validation_index) + profiled_names: set[str] = set() + for profile in profiles.values(): + if profile.get("type") in ("launcher", "alias"): + continue + for f in profile.get("files", []): + name = f.get("name", "") + if name: + profiled_names.add(name) + profiled_names.update(f.get("aliases") or []) dest_to_name: dict[str, str] = {} for sys_id, system in verify_systems.items(): for fe in system.get("files", []): @@ -845,6 +854,9 @@ def verify_platform( with_validation = sum( 1 for dest in file_status if dest_to_name.get(dest, "") in gt_filenames ) + with_profile = sum( + 1 for dest in file_status if dest_to_name.get(dest, "") in profiled_names + ) total = len(file_status) return { @@ -858,6 +870,7 @@ def verify_platform( "details": details, "ground_truth_coverage": { "with_validation": with_validation, + "with_profile": with_profile, "platform_only": total - with_validation, "total": total, "applicable": bool(resolve_platform_cores(config, profiles)),