mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-21 00:02:36 -05:00
feat: attach ground truth to emulator verification results
This commit is contained in:
+18
-1
@@ -715,6 +715,7 @@ def verify_emulator(
|
|||||||
details = []
|
details = []
|
||||||
file_status: dict[str, str] = {}
|
file_status: dict[str, str] = {}
|
||||||
file_severity: dict[str, str] = {}
|
file_severity: dict[str, str] = {}
|
||||||
|
dest_to_name: dict[str, str] = {}
|
||||||
data_dir_notices: list[str] = []
|
data_dir_notices: list[str] = []
|
||||||
|
|
||||||
for emu_name, profile in selected:
|
for emu_name, profile in selected:
|
||||||
@@ -736,6 +737,7 @@ def verify_emulator(
|
|||||||
"name": f"({emu_name})", "status": Status.OK,
|
"name": f"({emu_name})", "status": Status.OK,
|
||||||
"required": False, "system": "",
|
"required": False, "system": "",
|
||||||
"note": f"No files needed for {profile.get('emulator', emu_name)}",
|
"note": f"No files needed for {profile.get('emulator', emu_name)}",
|
||||||
|
"ground_truth": [],
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -759,8 +761,10 @@ def verify_emulator(
|
|||||||
"required": required}
|
"required": required}
|
||||||
result["system"] = file_entry.get("system", "")
|
result["system"] = file_entry.get("system", "")
|
||||||
result["hle_fallback"] = False
|
result["hle_fallback"] = False
|
||||||
|
result["ground_truth"] = build_ground_truth(archive, validation_index)
|
||||||
details.append(result)
|
details.append(result)
|
||||||
dest = archive
|
dest = archive
|
||||||
|
dest_to_name[dest] = archive
|
||||||
cur = result["status"]
|
cur = result["status"]
|
||||||
prev = file_status.get(dest)
|
prev = file_status.get(dest)
|
||||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
||||||
@@ -798,10 +802,12 @@ def verify_emulator(
|
|||||||
|
|
||||||
result["system"] = file_entry.get("system", "")
|
result["system"] = file_entry.get("system", "")
|
||||||
result["hle_fallback"] = hle
|
result["hle_fallback"] = hle
|
||||||
|
result["ground_truth"] = build_ground_truth(name, validation_index)
|
||||||
details.append(result)
|
details.append(result)
|
||||||
|
|
||||||
# Aggregate by destination (path if available, else name)
|
# Aggregate by destination (path if available, else name)
|
||||||
dest = file_entry.get("path", "") or name
|
dest = file_entry.get("path", "") or name
|
||||||
|
dest_to_name[dest] = name
|
||||||
cur = result["status"]
|
cur = result["status"]
|
||||||
prev = file_status.get(dest)
|
prev = file_status.get(dest)
|
||||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
||||||
@@ -820,14 +826,25 @@ def verify_emulator(
|
|||||||
|
|
||||||
label = _effective_validation_label(details, validation_index)
|
label = _effective_validation_label(details, validation_index)
|
||||||
|
|
||||||
|
gt_filenames = set(validation_index)
|
||||||
|
total = len(file_status)
|
||||||
|
with_validation = sum(
|
||||||
|
1 for dest in file_status if dest_to_name.get(dest, "") in gt_filenames
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"emulators": [n for n, _ in selected],
|
"emulators": [n for n, _ in selected],
|
||||||
"verification_mode": label,
|
"verification_mode": label,
|
||||||
"total_files": len(file_status),
|
"total_files": total,
|
||||||
"severity_counts": counts,
|
"severity_counts": counts,
|
||||||
"status_counts": status_counts,
|
"status_counts": status_counts,
|
||||||
"details": details,
|
"details": details,
|
||||||
"data_dir_notices": sorted(set(data_dir_notices)),
|
"data_dir_notices": sorted(set(data_dir_notices)),
|
||||||
|
"ground_truth_coverage": {
|
||||||
|
"with_validation": with_validation,
|
||||||
|
"platform_only": total - with_validation,
|
||||||
|
"total": total,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1433,6 +1433,23 @@ class TestE2E(unittest.TestCase):
|
|||||||
self.assertEqual(gt["platform_only"], gt["total"] - gt["with_validation"])
|
self.assertEqual(gt["platform_only"], gt["total"] - gt["with_validation"])
|
||||||
self.assertGreaterEqual(gt["with_validation"], 1)
|
self.assertGreaterEqual(gt["with_validation"], 1)
|
||||||
|
|
||||||
|
def test_118_emulator_result_has_ground_truth(self):
|
||||||
|
"""verify_emulator attaches ground_truth to each detail entry."""
|
||||||
|
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
|
||||||
|
for d in result["details"]:
|
||||||
|
self.assertIn("ground_truth", d)
|
||||||
|
# present_req.bin should have ground truth from test_validation
|
||||||
|
for d in result["details"]:
|
||||||
|
if d["name"] == "present_req.bin":
|
||||||
|
self.assertTrue(len(d["ground_truth"]) >= 1)
|
||||||
|
break
|
||||||
|
|
||||||
|
def test_119_emulator_result_ground_truth_coverage(self):
|
||||||
|
"""verify_emulator includes ground truth coverage counts."""
|
||||||
|
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
|
||||||
|
gt = result["ground_truth_coverage"]
|
||||||
|
self.assertEqual(gt["total"], result["total_files"])
|
||||||
|
|
||||||
def test_115_platform_result_ground_truth_empty_for_unknown(self):
|
def test_115_platform_result_ground_truth_empty_for_unknown(self):
|
||||||
"""Files with no emulator validation get ground_truth=[]."""
|
"""Files with no emulator validation get ground_truth=[]."""
|
||||||
config = load_platform_config("test_existence", self.platforms_dir)
|
config = load_platform_config("test_existence", self.platforms_dir)
|
||||||
|
|||||||
Reference in New Issue
Block a user