From 58735286a7d931bb0bf51148f2d506db8949b1f9 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:26:05 +0100 Subject: [PATCH] feat: let an archive join a set Kronos reads kronos/stvbios.zip, else stvbios.zip at the root of the system directory, and a ST-V game stops without either (libretro.c:1608-1617, 1676-1681 at the pin). The archive under archive_prefix and the root copy are one need, but only loose entries could be members, so the required-only pack carried both. An archive's own entry now joins a set like a loose file. verify --emulator and the emulator pack settle archive units with the loose entries, the required-only pick covers them, find_undeclared_files and cross_reference record them, and the unit under archive_prefix ranks before the same name at the root. Its members still settle the archive by their contents. Kronos' stvbios.zip is grouped stv-bios. No other profile reads one archive under two paths: odin and tgpulse_next_m1 place their own entries at the prefix path, and flycast's naomi.zip has one path per build. --- emulators/kronos.yml | 6 ++- scripts/alternatives.py | 7 +++- scripts/cross_reference.py | 21 ++++++++-- scripts/generate_pack.py | 35 ++++++++++------ scripts/packextras.py | 47 ++++++++++++++------- scripts/verify.py | 52 +++++++++++++---------- tests/test_e2e.py | 86 ++++++++++++++++++++++++++++++++++++++ wiki/profiling.md | 2 +- 8 files changed, 201 insertions(+), 55 deletions(-) diff --git a/emulators/kronos.yml b/emulators/kronos.yml index 8090a1923..16ed978ee 100644 --- a/emulators/kronos.yml +++ b/emulators/kronos.yml @@ -51,6 +51,8 @@ archive_prefix: kronos groups: saturn-bios: required: true + stv-bios: + required: true files: # ----------------------------------------------------------- @@ -104,8 +106,8 @@ files: # ----------------------------------------------------------- - name: "stvbios.zip" system: sega-stv - required: true - note: "MAME-style ZIP containing ST-V region BIOS EPROMs. Searched in kronos/ subdir first." + group: stv-bios + note: "MAME-style ZIP containing ST-V region BIOS EPROMs. Read as kronos/stvbios.zip, else at the root of the system directory, and a ST-V game stops without either (libretro.c:1676-1681)." source_ref: "source yabause/src/libretro/libretro.c:1608-1617, source yabause/src/ctrl/src/stv.c:112-136" # ----------------------------------------------------------- diff --git a/scripts/alternatives.py b/scripts/alternatives.py index 3893981ff..e083e4a79 100644 --- a/scripts/alternatives.py +++ b/scripts/alternatives.py @@ -197,7 +197,12 @@ def core_file_spares( and verify alike. """ answered = {id(e) for e in entries if e.get("set_declared")} - rest = [e for e in entries if id(e) not in answered] + # An archive under the core's own folder is read before the same name + # at the root (Kronos), so archive units rank first among equals. + rest = sorted( + (e for e in entries if id(e) not in answered), + key=lambda e: not e.get("archive"), + ) return answered | required_only_spares( rest, owner=lambda e: e.get("source_profile") or e.get("profile") or "", diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index f48d2f5a4..d5af100b4 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -801,6 +801,7 @@ def _cross_reference_profile( # Grouped entries met, as (set keys, entry, fills, the list holding # its row, the row): settled once the profile is read. set_records: list[tuple[tuple, dict, bool, list | None, dict | None]] = [] + archive_rows: list[dict] = [] def fate(f: dict, what: str) -> None: key = position[id(f)] @@ -908,6 +909,15 @@ def _cross_reference_profile( "archive_required_count": 0, } entry = archive_gaps[archive_key] + if entry["archive_file_count"] == 0: + # The archive's own declaration names its sets: an archive + # is a member like a loose file (Kronos' stvbios.zip). + own = archive_entry(archive, emu_files) + set_records.append(( + set_keys((emu_name, is_standalone), own), own, + entry["in_repo"], archive_rows, entry, + )) + archive_rows.append(entry) entry["archive_file_count"] += 1 if f.get("required", False): entry["archive_required_count"] += 1 @@ -968,7 +978,10 @@ def _cross_reference_profile( # One need per set: a member another one fills is not a gap, and a set # nobody holds is one gap under its first member (alternatives.py), as # verify and the packs count it. - grouped = [r for r in set_records if r[0]] + # Archive units rank first among equals, as the builder orders them. + grouped = sorted( + (r for r in set_records if r[0]), key=lambda r: r[3] is not archive_rows + ) requirements = set_requirements(profile) excused, standing = settle_sets( [(keys, f, fills) for keys, f, fills, _l, _r in grouped], @@ -982,10 +995,12 @@ def _cross_reference_profile( set_row["required"] = standing[at].required if at in excused: rows.remove(set_row) - fates[position[id(member)]] = "other_versions" + if id(member) in position: + fates[position[id(member)]] = "other_versions" for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): - gaps.append(ag) + if any(ag is row for row in archive_rows): + gaps.append(ag) tally = Counter(fates.values()) report[emu_name] = { diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index a6551c33c..559889acc 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1786,6 +1786,16 @@ def generate_emulator_pack( if archive: archives.add(archive) + # What each destination came to, settled by set once the + # profile is read: a member another member fills is not missing, + # and a set nobody holds is reported once (alternatives.py). An + # archive unit is a member like a loose file. + outcomes: list[tuple[list[dict], bool, Callable[[], None] | None]] = [] + spares = emulator_required_spares( + emu_name, profile, files, standalone, region_drops, db, + validation_index, zip_contents, data_registry, + ) if required_only else set() + # Pack archives as units archive_prefix = profile.get("archive_prefix", "") for archive_name in sorted(archives): @@ -1796,6 +1806,8 @@ def generate_emulator_pack( own_entry = archive_entry(archive_name, files) if required_only and archive_is_optional(own_entry, archive_name, files): continue + if archive_dest in spares: + continue local_path, status = resolve_file( own_entry, db, @@ -1814,7 +1826,11 @@ def generate_emulator_pack( if not (taken and local_path and _same_bytes( local_path, packed_at.get(archive_dest.lower()) )): - report_missing(archive_name, archive_dest) + outcomes.append(([own_entry], False, functools.partial( + report_missing, archive_name, archive_dest + ))) + else: + outcomes.append(([own_entry], True, None)) continue if local_path and status not in ("not_found",): if local_path.endswith(".zip"): @@ -1828,35 +1844,30 @@ def generate_emulator_pack( total_files += 1 if record is not None: record.files.append(archive_dest) + outcomes.append(([own_entry], True, None)) else: - report_missing(archive_name, archive_dest) + outcomes.append(([own_entry], False, functools.partial( + report_missing, archive_name, archive_dest + ))) # Pack individual files (skip archive members). The entries of one # profile on one destination are the releases the code accepts # there (raze's seven SW.GRP): the first one the core accepts # fills the slot, as verify --emulator credits it, and the slot # has one outcome however many alternatives it lists. - spares = emulator_required_spares( - emu_name, profile, files, standalone, region_drops, db, - validation_index, zip_contents, data_registry, - ) if required_only else set() slots: dict[str, list[dict]] = {} for fe in files: - if required_only and (fe.get("required") is False or id(fe) in spares): + if required_only and fe.get("required") is False: continue if fe.get("archive"): continue dest = _resolve_destination(fe, standalone) - if not dest: + if not dest or dest in spares: continue if region_drops and dest in region_drops: continue slots.setdefault(dest, []).append(fe) - # What each destination came to, settled by set once the loop - # is over: a member another member fills is not missing, and a - # set nobody holds is reported once (alternatives.py). - outcomes: list[tuple[list[dict], bool, Callable[[], None] | None]] = [] for dest, alternatives in slots.items(): # Name order, first file wins, case folded: verify replays # the rule and reports a later profile's file as missing. diff --git a/scripts/packextras.py b/scripts/packextras.py index 8985a23c6..c4f315d69 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -1881,26 +1881,43 @@ def emulator_required_spares( validation_index: dict, zip_contents: dict | None, data_registry: dict | None, -) -> set[int]: - """The members of required sets an emulator --required-only pack leaves - out: one held member stays per set, the first the code loads among those - the pack still carries after the region filter. The pack and - verify --emulator both read this.""" +) -> set[str]: + """The destinations an emulator --required-only pack leaves out: of each + required set, one held member stays, the first the code loads among + those the pack still carries after the region filter. An archive unit + is a member like a loose file: Kronos reads kronos/stvbios.zip, else + stvbios.zip. The pack and verify --emulator both read this.""" from alternatives import groups_of, rank_in, required_groups - from packpaths import _resolve_destination + from common import archive_entry + from packpaths import _archive_destination, _resolve_destination - loose = [fe for fe in files if not fe.get("archive") and groups_of(fe)] - if not loose: + candidates: list[dict] = [] + archives: list[str] = [] + for fe in files: + archive = fe.get("archive") + if archive and archive not in archives: + archives.append(archive) + own = archive_entry(archive, files) + if groups_of(own): + candidates.append({"dest": _archive_destination( + archive, profile.get("archive_prefix", ""), + profile.get("pack_structure"), standalone, + ), "entry": own}) + for fe in files: + if not fe.get("archive") and groups_of(fe): + candidates.append({"dest": _resolve_destination(fe, standalone), "entry": fe}) + if not candidates: return set() - return required_only_spares( - loose, - owner=lambda _fe: emu_name, - held=lambda fe: _emulator_ships( - fe, emu_name, profile, db, validation_index, zip_contents, data_registry, + spares = required_only_spares( + candidates, + owner=lambda _c: emu_name, + held=lambda c: _emulator_ships( + c["entry"], emu_name, profile, db, validation_index, zip_contents, data_registry, ), - gone=lambda fe: _resolve_destination(fe, standalone) in region_drops, - sets=lambda fe: {g: rank_in(fe, g) for g in required_groups(fe, profile)}, + gone=lambda c: c["dest"] in region_drops, + sets=lambda c: {g: rank_in(c["entry"], g) for g in required_groups(c["entry"], profile)}, ) + return {c["dest"] for c in candidates if id(c) in spares} def _emulator_ships( diff --git a/scripts/verify.py b/scripts/verify.py index 833d2a179..f91b83359 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -665,6 +665,14 @@ def find_undeclared_files( } if f.get("system_dir"): archive_entries[archive_key]["system_dir"] = f["system_dir"] + # The archive's own declaration names its sets: an + # archive is a member like a loose file. + own = archive_entry(archive, profile.get("files", [])) + archive_entries[archive_key].update(set_fields(own, profile)) + set_records.append(( + set_keys((emu_name, is_standalone), own), own, in_repo, + (archive_entries[archive_key], None), + )) entry = archive_entries[archive_key] entry["archive_file_count"] += 1 if f.get("required", False): @@ -793,12 +801,10 @@ def find_undeclared_files( undeclared = [u for u in undeclared if id(u) not in settled] undeclared.extend(added) - undeclared = _settle_core_sets(undeclared, set_records, settled, profiles) - for entry in sorted(archive_entries.values(), key=lambda e: e["name"]): undeclared.append(entry) - return undeclared + return _settle_core_sets(undeclared, set_records, settled, profiles) def _settle_core_sets( @@ -814,7 +820,11 @@ def _settle_core_sets( holds keeps one entry, its first member, naming the others. A claim another profile shares stays: its own core may need that very file. """ - grouped = [r for r in records if r[0]] + # Archive units rank first among equals, as the builder orders them. + grouped = sorted( + (r for r in records if r[0]), + key=lambda r: not (r[3] is not None and r[3][0].get("archive")), + ) if not grouped: return undeclared excused, standing = settle_sets([ @@ -2365,6 +2375,7 @@ def verify_emulator( if fe.get("archive") or _resolve_destination(fe, standalone) not in region_drops ] + spares: set[str] = set() if required_only: from packextras import emulator_required_spares @@ -2372,7 +2383,10 @@ def verify_emulator( emu_name, profile, files, standalone, region_drops, db, validation_index, zip_contents, data_registry, ) - files = [fe for fe in files if id(fe) not in spares] + files = [ + fe for fe in files + if fe.get("archive") or _resolve_destination(fe, standalone) not in spares + ] # Check data directories (only notice if not cached) for dd in deliverable( @@ -2403,6 +2417,12 @@ def verify_emulator( occupy_trees(emu_name, profile) continue + # Each destination's outcome, settled by set once the profile is + # read: an archive unit is a member like a loose file (Kronos reads + # kronos/stvbios.zip, else stvbios.zip). + placed_results: list[tuple[str, dict, bool, bool, list[dict]]] = [] + fallback_reason = "declared hash contradicted by the local file" + # Verify archives as units (e.g., neogeo.zip, aes.zip) seen_archives: set[str] = set() for file_entry in files: @@ -2410,6 +2430,11 @@ def verify_emulator( if archive and archive not in seen_archives: seen_archives.add(archive) own_entry = archive_entry(archive, files) + if _archive_destination( + archive, profile.get("archive_prefix", ""), + profile.get("pack_structure"), standalone, + ) in spares: + continue local_path, resolve_status = pending(own_entry, *resolve_file( own_entry, db, @@ -2452,7 +2477,6 @@ def verify_emulator( result["system"] = file_entry.get("system", "") result["hle_fallback"] = False result["ground_truth"] = build_ground_truth(archive, validation_index) - details.append(result) # Where the pack puts it: an archive_prefix or pack_structure # moves it, and a loose same-named entry is another file. dest = _archive_destination( @@ -2476,27 +2500,13 @@ def verify_emulator( elif local_path: occupy(dest, local_path, emu_name) packed_dests.add(dest) - dest_to_name[dest] = archive - cur = result["status"] - prev = file_status.get(dest) - if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get( - prev, 0 - ): - file_status[dest] = cur - sev = _emulator_severity(cur, required, False) - prev_sev = file_severity.get(dest) - if prev_sev is None or _SEVERITY_ORDER.get( - sev, 0 - ) > _SEVERITY_ORDER.get(prev_sev, 0): - file_severity[dest] = sev + placed_results.append((dest, result, required, False, [own_entry])) # Results of this emulator, by destination. Several entries of one # profile on one destination are the releases the code accepts # there (a shareware and a retail HOG, each GRP in a size table): # holding one of them fills the slot, and the others are not gaps. emu_results: dict[str, list[tuple[dict, bool, bool, dict]]] = {} - placed_results: list[tuple[str, dict, bool, bool, list[dict]]] = [] - fallback_reason = "declared hash contradicted by the local file" for file_entry in files: # Skip archive members (verified as archive units above) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9907b92a9..aa943c7ac 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -7033,5 +7033,91 @@ class OneOfASearchList(unittest.TestCase): kept = sorted(u["name"] for u in undeclared if id(u) not in spares) self.assertEqual(kept, ["none_a.bin", "plain.bin", "second.bin"]) +class AnArchiveIsAMemberOfASet(unittest.TestCase): + """Kronos reads kronos/stvbios.zip, else stvbios.zip at the root: one + archive under two paths, a first-found pair. The archive unit and the + root copy settle as members of one set, and a --required-only pack + carries the first, while the members inside still settle the archive + by its contents.""" + + def _fixture(self, held: bool) -> dict: + import zlib + + from common import _emulator_profiles_cache + + _emulator_profiles_cache.clear() + repo = Path(__file__).resolve().parent.parent + (repo / "tmp").mkdir(exist_ok=True) + root = Path(tempfile.mkdtemp(dir=repo / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + previous = os.getcwd() + os.chdir(root) + self.addCleanup(os.chdir, previous) + self.db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}, + "by_crc32": {}, "by_path_suffix": {}}} + rom = b"STV" * 64 + if held: + rel = "bios/Sega/STV/set.zip" + Path(rel).parent.mkdir(parents=True) + with zipfile.ZipFile(rel, "w") as zf: + zf.writestr("epr.ic8", rom) + data = Path(rel).read_bytes() + sha1 = hashlib.sha1(data).hexdigest() + self.db["files"][sha1] = {"path": rel, "name": "set.zip", "size": len(data), + "md5": hashlib.md5(data).hexdigest(), "crc32": ""} + self.db["indexes"]["by_name"]["set.zip"] = [sha1] + Path("emulators").mkdir() + Path("platforms").mkdir() + Path("emulators/stv.yml").write_text(yaml.safe_dump({ + "emulator": "Stv", "type": "libretro", "cores": ["stv"], + "systems": ["sega-stv"], "archive_prefix": "stv", + "groups": {"stv-bios": {"required": True}}, + "files": [ + {"name": "set.zip", "system": "sega-stv", "group": "stv-bios"}, + {"name": "epr.ic8", "archive": "set.zip", "system": "sega-stv", + "required": True, "crc32": f"{zlib.crc32(rom):08x}"}, + ], + })) + return load_emulator_profiles("emulators")["stv"] + + def test_the_archive_and_its_root_copy_are_one_need(self): + self._fixture(held=False) + result = verify_emulator(["stv"], "emulators", self.db, platforms_dir="platforms") + missing = [d for d in result["details"] if d["status"] == "missing"] + self.assertEqual([d["destination"] for d in missing], ["stv/set.zip"]) + self.assertEqual(missing[0]["alternatives"], ["set.zip"]) + self.assertEqual(result["status_counts"].get("missing"), 1) + + def test_required_only_carries_the_archive_the_code_reads_first(self): + from generate_pack import generate_emulator_pack + + self._fixture(held=True) + full = verify_emulator(["stv"], "emulators", self.db, platforms_dir="platforms") + self.assertEqual( + sorted(d["destination"] for d in full["details"] if d["status"] == "ok"), + ["set.zip", "stv/set.zip"], + ) + result = verify_emulator(["stv"], "emulators", self.db, + platforms_dir="platforms", required_only=True) + held = sorted(d["destination"] for d in result["details"] if d["status"] == "ok") + self.assertEqual(held, ["stv/set.zip"]) + with contextlib.redirect_stdout(io.StringIO()): + zip_path = generate_emulator_pack( + ["stv"], "emulators", self.db, "bios", "out", offline=True, + platforms_dir="platforms", required_only=True, + ) + with zipfile.ZipFile(zip_path) as zf: + packed = sorted(n for n in zf.namelist() if n != "README.txt") + self.assertEqual(packed, held) + + def test_core_files_report_the_pair_once(self): + profile = self._fixture(held=False) + config = {"platform": "Stvs", "verification_mode": "existence", + "base_destination": "system", "cores": ["stv"], + "systems": {"sega-stv": {"files": []}}} + undeclared = find_undeclared_files(config, "emulators", self.db, {"stv": profile}) + self.assertEqual([u["name"] for u in undeclared], ["set.zip"]) + self.assertTrue(undeclared[0].get("archive")) + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index 4bf3697c8..69e6ea50e 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -622,7 +622,7 @@ look. | `region_check` | true if the core refuses to boot a game whose region does not match the BIOS | | `fast_boot` | BIOS generation label the core uses to decide whether fast boot is available | | `priority` | preference order when several BIOS satisfy the same slot, **lowest wins**. DuckStation's `FindBIOSImageInDirectory` keeps the image whose priority is lower, and its table de-prioritizes by raising the number: the buggy launch console sits at 50 and PS2 images at 100, while `scph5501` sits at 5. A core that walks an ordered search list, as PicoDrive does with `biosfiles_us/eu/jp`, maps onto the same field as 1, 2, 3. Only set it from the source. `--one-per-slot` reads it, comparing the members of one `group` | -| `group` | the sets of the profile's `groups:` map an entry belongs to. A set is one need: the code loads one member (the first it finds, as Yabause tries `saturn_bios.bin`, `sega_101.bin`, `mpr-17933.bin`) and holding any member fills it. Verify, the gap report and the packs report a set nobody holds as one gap under its first member, and no member of a held set as missing. A `--required-only` build keeps one member per required set, the first the code reads among those held. A list puts a file in several sets, a map from set to rank gives its place in each when it differs (pom2's `apple2.rom` is second for the ][ and third at boot). Where the code keeps a list per region, each list is its own set (PicoDrive's `mega-cd-us`, `mega-cd-eu`, `mega-cd-jp`) | +| `group` | the sets of the profile's `groups:` map an entry belongs to. A set is one need: the code loads one member (the first it finds, as Yabause tries `saturn_bios.bin`, `sega_101.bin`, `mpr-17933.bin`) and holding any member fills it. Verify, the gap report and the packs report a set nobody holds as one gap under its first member, and no member of a held set as missing. A `--required-only` build keeps one member per required set, the first the code reads among those held. A list puts a file in several sets, a map from set to rank gives its place in each when it differs (pom2's `apple2.rom` is second for the ][ and third at boot). Where the code keeps a list per region, each list is its own set (PicoDrive's `mega-cd-us`, `mega-cd-eu`, `mega-cd-jp`). An archive's own entry joins a set like a loose file, its unit under `archive_prefix` ranking before the same name at the root (Kronos reads `kronos/stvbios.zip`, else `stvbios.zip`), and its members still settle it by their contents | | `groups` (profile) | the requirement of each set, `{name: {required: bool}}`. A member writes no `required`: it is required when one of its sets is, so a fallback shared by a required set and an optional one needs no value of its own | | `embedded` | true if the data is compiled into the binary and the external file is optional | | `bundled` | true if the emulator's own release installs the file where it reads it, for the build `mode` names (all builds when `mode` is absent). Packs and the installer leave it out: the user already has it. A file the standalone release ships but the libretro core does not (PPSSPP's assets, Dolphin's Sys) is bundled with `mode: standalone` only |