From 8c018849ae5c5b3dbf85e0def9234cb5a8064922 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:41:04 +0200 Subject: [PATCH] fix: return instead of continue in the split profile The cross-reference loop body became its own function, but one exit stayed a continue and left the module unparseable, which took the whole test suite down with it. The other ten continues are inside genuine inner loops and stand. Output verified identical to the version before the split. --- scripts/cross_reference.py | 435 ++++++++++++++++++++----------------- 1 file changed, 236 insertions(+), 199 deletions(-) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index a670741c..379525b1 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -180,6 +180,229 @@ def _resolve_archive_source( return result if result is not None else "missing" +def _cross_reference_profile( + emu_name: str, + profile: dict, + declared: dict[str, set[str]], + report: dict, + index: dict, +) -> None: + """Compare one emulator profile against what the platforms declare. + + Split out of cross_reference, where it was a 200-line loop body that + carried the whole function's branching: every file is resolved by name, + by path, by hash and by archive membership before it can be called a + gap. *index* carries the database lookups those steps share. + """ + by_name = index["by_name"] + by_name_lower = index["by_name_lower"] + by_md5 = index["by_md5"] + by_crc32 = index["by_crc32"] + by_path_suffix = index["by_path_suffix"] + db_files = index["db_files"] + data_names = index["data_names"] + all_declared = index["all_declared"] + emu_files = profile.get("files", []) + systems = profile.get("systems", []) + + # Skip filename-agnostic profiles (BIOS detected without fixed names) + if profile.get("bios_mode") == "agnostic": + return + + if all_declared is not None: + platform_names = all_declared + else: + platform_names = set() + for sys_id in systems: + platform_names.update(declared.get(sys_id, set())) + + gaps = [] + covered = [] + unsourceable_list: list[dict] = [] + archive_gaps: dict[tuple, dict] = {} + seen_files: set[tuple] = set() + for f in emu_files: + fname = f.get("name", "") + effective_path = f.get("path") or fname + seen_key = ( + fname, + f.get("archive"), + effective_path, + f.get("system"), + f.get("variant_group"), + f.get("mode", "both"), + ) + if not fname or seen_key in seen_files: + continue + + # Collect unsourceable files separately (documented, not a gap) + unsourceable_reason = f.get("unsourceable", "") + if unsourceable_reason: + seen_files.add(seen_key) + unsourceable_list.append({ + "name": fname, + "required": f.get("required", False), + "reason": unsourceable_reason, + "source_ref": f.get("source_ref", ""), + }) + continue + + # Skip pattern placeholders (e.g., .bin, .bin) + if "<" in fname or ">" in fname or "*" in fname: + continue + + # Skip UI-imported files with explicit path: null (not resolvable by pack) + if "path" in f and f["path"] is None: + continue + + # Skip standalone-only files + file_mode = f.get("mode", "both") + if file_mode == "standalone": + continue + + # Skip files loaded from non-system directories (save_dir, content_dir) + load_from = f.get("load_from", "") + if load_from and load_from != "system_dir": + continue + + # Skip filename-agnostic files (handled by agnostic scan) + if f.get("agnostic"): + continue + + archive = f.get("archive") + + # Check platform declaration (by name or archive) + in_platform = fname in platform_names + if not in_platform and archive: + in_platform = archive in platform_names + + if in_platform: + seen_files.add(seen_key) + covered.append({ + "name": fname, + "path": effective_path, + "required": f.get("required", False), + "in_platform": True, + }) + continue + + seen_files.add(seen_key) + + # Group archived files by archive name + if archive: + archive_key = ( + archive, + f.get("system"), + f.get("variant_group"), + f.get("mode", "both"), + ) + if archive_key not in archive_gaps: + source = _resolve_archive_source( + archive, by_name, by_name_lower, data_names, + by_path_suffix, + ) + archive_gaps[archive_key] = { + "name": archive, + "path": archive, + "required": False, + "note": "", + "source_ref": "", + "in_platform": False, + "in_repo": source != "missing", + "source": source, + "archive": archive, + "archive_file_count": 0, + "archive_required_count": 0, + } + entry = archive_gaps[archive_key] + entry["archive_file_count"] += 1 + if f.get("required", False): + entry["archive_required_count"] += 1 + entry["required"] = True + if not entry["source_ref"] and f.get("source_ref"): + entry["source_ref"] = f["source_ref"] + continue + + # --- resolve source provenance --- + storage = f.get("storage", "") + if storage in ("release", "large_file"): + source = "large_file" + else: + source = _resolve_source( + fname, by_name, by_name_lower, data_names, by_path_suffix, + f, db_files, + ) + if source is None: + path_field = f.get("path", "") + if path_field and path_field != fname: + source = _resolve_source( + path_field, by_name, by_name_lower, + data_names, by_path_suffix, f, db_files, + ) + # Try the alternate names the emulator accepts, like + # resolve_local_file does + if source is None: + for alias in f.get("aliases") or []: + source = _resolve_source( + alias, by_name, by_name_lower, + data_names, by_path_suffix, f, db_files, + ) + if source is not None: + break + # Try MD5 hash match + if source is None: + for md5_val in parse_md5_list(f.get("md5")): + if by_md5.get(md5_val): + source = "bios" + break + # Try SHA1 hash match + if source is None: + raw_sha1 = f.get("sha1", "") + sha1_values = raw_sha1 if isinstance(raw_sha1, list) else [raw_sha1] + if any(value and value in db_files for value in sha1_values): + source = "bios" + # Try CRC32 hash match + if source is None: + crc32 = str(f.get("crc32", "")).lower() + if crc32 and by_crc32.get(crc32): + source = "bios" + if source is None: + source = "missing" + + in_repo = source != "missing" + + entry = { + "name": fname, + "path": effective_path, + "required": f.get("required", False), + "note": f.get("note", ""), + "source_ref": f.get("source_ref", ""), + "in_platform": False, + "in_repo": in_repo, + "source": source, + } + gaps.append(entry) + + # Append grouped archive gaps + for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): + gaps.append(ag) + + report[emu_name] = { + "emulator": profile.get("emulator", emu_name), + "systems": systems, + "total_files": len(emu_files), + "platform_covered": len(covered), + "gaps": len(gaps), + "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), + "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), + "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), + "gap_data": sum(1 for g in gaps if g["source"] == "data"), + "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), + "gap_details": gaps, + "unsourceable": unsourceable_list, + } + + def cross_reference( profiles: dict[str, dict], declared: dict[str, set[str]], @@ -212,206 +435,20 @@ def cross_reference( db_files = db.get("files", {}) report = {} + index = { + "by_name": by_name, + "by_name_lower": by_name_lower, + "by_md5": by_md5, + "by_crc32": by_crc32, + "by_path_suffix": by_path_suffix, + "db_files": db_files, + "data_names": data_names, + "all_declared": all_declared, + } for emu_name, profile in profiles.items(): - emu_files = profile.get("files", []) - systems = profile.get("systems", []) - - # Skip filename-agnostic profiles (BIOS detected without fixed names) - if profile.get("bios_mode") == "agnostic": - continue - - if all_declared is not None: - platform_names = all_declared - else: - platform_names = set() - for sys_id in systems: - platform_names.update(declared.get(sys_id, set())) - - gaps = [] - covered = [] - unsourceable_list: list[dict] = [] - archive_gaps: dict[tuple, dict] = {} - seen_files: set[tuple] = set() - for f in emu_files: - fname = f.get("name", "") - effective_path = f.get("path") or fname - seen_key = ( - fname, - f.get("archive"), - effective_path, - f.get("system"), - f.get("variant_group"), - f.get("mode", "both"), - ) - if not fname or seen_key in seen_files: - continue - - # Collect unsourceable files separately (documented, not a gap) - unsourceable_reason = f.get("unsourceable", "") - if unsourceable_reason: - seen_files.add(seen_key) - unsourceable_list.append({ - "name": fname, - "required": f.get("required", False), - "reason": unsourceable_reason, - "source_ref": f.get("source_ref", ""), - }) - continue - - # Skip pattern placeholders (e.g., .bin, .bin) - if "<" in fname or ">" in fname or "*" in fname: - continue - - # Skip UI-imported files with explicit path: null (not resolvable by pack) - if "path" in f and f["path"] is None: - continue - - # Skip standalone-only files - file_mode = f.get("mode", "both") - if file_mode == "standalone": - continue - - # Skip files loaded from non-system directories (save_dir, content_dir) - load_from = f.get("load_from", "") - if load_from and load_from != "system_dir": - continue - - # Skip filename-agnostic files (handled by agnostic scan) - if f.get("agnostic"): - continue - - archive = f.get("archive") - - # Check platform declaration (by name or archive) - in_platform = fname in platform_names - if not in_platform and archive: - in_platform = archive in platform_names - - if in_platform: - seen_files.add(seen_key) - covered.append({ - "name": fname, - "path": effective_path, - "required": f.get("required", False), - "in_platform": True, - }) - continue - - seen_files.add(seen_key) - - # Group archived files by archive name - if archive: - archive_key = ( - archive, - f.get("system"), - f.get("variant_group"), - f.get("mode", "both"), - ) - if archive_key not in archive_gaps: - source = _resolve_archive_source( - archive, by_name, by_name_lower, data_names, - by_path_suffix, - ) - archive_gaps[archive_key] = { - "name": archive, - "path": archive, - "required": False, - "note": "", - "source_ref": "", - "in_platform": False, - "in_repo": source != "missing", - "source": source, - "archive": archive, - "archive_file_count": 0, - "archive_required_count": 0, - } - entry = archive_gaps[archive_key] - entry["archive_file_count"] += 1 - if f.get("required", False): - entry["archive_required_count"] += 1 - entry["required"] = True - if not entry["source_ref"] and f.get("source_ref"): - entry["source_ref"] = f["source_ref"] - continue - - # --- resolve source provenance --- - storage = f.get("storage", "") - if storage in ("release", "large_file"): - source = "large_file" - else: - source = _resolve_source( - fname, by_name, by_name_lower, data_names, by_path_suffix, - f, db_files, - ) - if source is None: - path_field = f.get("path", "") - if path_field and path_field != fname: - source = _resolve_source( - path_field, by_name, by_name_lower, - data_names, by_path_suffix, f, db_files, - ) - # Try the alternate names the emulator accepts, like - # resolve_local_file does - if source is None: - for alias in f.get("aliases") or []: - source = _resolve_source( - alias, by_name, by_name_lower, - data_names, by_path_suffix, f, db_files, - ) - if source is not None: - break - # Try MD5 hash match - if source is None: - for md5_val in parse_md5_list(f.get("md5")): - if by_md5.get(md5_val): - source = "bios" - break - # Try SHA1 hash match - if source is None: - raw_sha1 = f.get("sha1", "") - sha1_values = raw_sha1 if isinstance(raw_sha1, list) else [raw_sha1] - if any(value and value in db_files for value in sha1_values): - source = "bios" - # Try CRC32 hash match - if source is None: - crc32 = str(f.get("crc32", "")).lower() - if crc32 and by_crc32.get(crc32): - source = "bios" - if source is None: - source = "missing" - - in_repo = source != "missing" - - entry = { - "name": fname, - "path": effective_path, - "required": f.get("required", False), - "note": f.get("note", ""), - "source_ref": f.get("source_ref", ""), - "in_platform": False, - "in_repo": in_repo, - "source": source, - } - gaps.append(entry) - - # Append grouped archive gaps - for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): - gaps.append(ag) - - report[emu_name] = { - "emulator": profile.get("emulator", emu_name), - "systems": systems, - "total_files": len(emu_files), - "platform_covered": len(covered), - "gaps": len(gaps), - "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), - "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), - "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), - "gap_data": sum(1 for g in gaps if g["source"] == "data"), - "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), - "gap_details": gaps, - "unsourceable": unsourceable_list, - } + _cross_reference_profile( + emu_name, profile, declared, report, index + ) return report