chore: merge launcher names

This commit is contained in:
Abdessamad Derraz committed 2026-11-09 18:29:52 +01:00
commit 2fe15e99f9
24 files changed
+1400 -62

No files matched your search

+44
View File
@@ -3581,6 +3581,50 @@ def declared_root(config: dict) -> str:
return root if count / len(roots) > 0.9 else ""
def platform_declared_contents(config: dict, db: dict) -> dict[str, set[str]]:
"""The folders each declared file's content sits in, keyed by its hashes.
A core that scans a folder and identifies a ROM by its checksum reads a
declared file whatever its name: RetroDECK lists atari800's Atari BASIC
Rev A as ROM_BASIC_A, and the core takes it by CRC32. Keys are
"md5:", "sha1:" and "crc32:" values of the database file the
declaration resolves to, each with the declared slot's folder.
"""
prefix = declared_root(config).lower()
by_md5 = db.get("indexes", {}).get("by_md5", {})
files_db = db.get("files", {})
contents: dict[str, set[str]] = {}
for system in config.get("systems", {}).values():
for fe in system.get("files", []):
if fe.get("zipped_file"):
continue
record = files_db.get((fe.get("sha1") or "").lower())
md5 = (fe.get("md5") or "").lower()
if record is None and md5 and "," not in md5:
sha1 = by_md5.get(md5, "")
record = files_db.get(sha1[0] if isinstance(sha1, list) else sha1)
if not record:
continue
slot = sanitize_pack_path(fe.get("destination") or fe.get("name", "")).lower()
if prefix and slot.startswith(prefix + "/"):
slot = slot[len(prefix) + 1:]
folder = slot.rpartition("/")[0]
for field in ("md5", "sha1", "crc32"):
value = str(record.get(field) or "").lower()
if value:
contents.setdefault(f"{field}:{value}", set()).add(folder)
return contents
def merged_platform_declared_contents(configs: list[dict], db: dict) -> dict[str, set[str]]:
"""`platform_declared_contents` of several platforms, one index."""
merged: dict[str, set[str]] = {}
for config in configs:
for key, folders in platform_declared_contents(config, db).items():
merged.setdefault(key, set()).update(folders)
return merged
def merged_platform_declarations(
configs: list[dict], db: dict
) -> dict[str, list[tuple[int | None, str]]]:
+106 -4
View File
@@ -35,6 +35,7 @@ from common import (
get_mame_clone_map,
list_emulator_profiles,
list_registered_platforms,
merged_platform_declared_contents,
load_data_dir_registry,
load_database,
load_emulator_profiles,
@@ -50,7 +51,17 @@ from common import (
select_emulator_profiles,
)
from hashing import parse_hash_list
from launchers import LauncherLayer, declared_destinations, held_record
from launchers import (
LauncherLayer,
catalogued_model_files,
declared_destinations,
held_record,
name_settings,
set_by,
setting_readers,
suffixed_stem,
)
from launchers import _directories as launcher_directories
from launchers import _names as launcher_names
from validation import (
case_folded_slots,
@@ -857,7 +868,8 @@ def _cross_reference_profile(
# A platform declares files of its BIOS tree: none answers for one
# the core reads below the save directory.
in_platform = not save_root and f.get("root") != "roms" and declaration_settles(
f, fname, platform_names, effective_path, index.get("folded_slots", frozenset())
f, fname, platform_names, effective_path, index.get("folded_slots", frozenset()),
index.get("declared_contents"),
)
if in_platform:
@@ -1119,8 +1131,11 @@ class _NameHolder:
return self._cache[name]
base = name.rsplit("/", 1)[-1]
found: tuple[str, bool] | None = None
stem = suffixed_stem(base)
if base in self.by_name or any(map(self._entry_held, self.exact.get(base, []))):
found = (base, True)
elif stem is not None and any(map(self._entry_held, self.exact.get(stem, []))):
found = (base, True)
else:
path, _status = resolve_local_file(
{"name": name}, self.db, data_dir_registry=self.registry
@@ -1155,6 +1170,16 @@ def launcher_reads_not_held(
a copy is reported with `held_as`, since the launcher does not open it
under the name it asks for. *emulator* keeps the records whose emulator
name designates that profile.
Three records ask for nothing the collection lacks. A search keeps the
first name it finds, so one held name fills it: RetroBat's Amiberry
launcher returns the first of kick34005.A500, kick34005.A500.rom and
amiga-os-130.rom that exists (Amiberry.Generator.cs:674-699). A copy
whose source lies outside the BIOS directory brings a file the platform
installs itself, from its own package or written by its own script
(RetroPie's lr-vecx copies $md_inst/system.bin). A file handed to the
setting a reader's profile entry is read from is that entry's file,
which the pack places under the launcher's name.
"""
holder = _NameHolder(db, profiles, data_dir_registry)
every_profile = set(profiles)
@@ -1170,9 +1195,45 @@ def launcher_reads_not_held(
for part in str(record.get("emulator", "")).split(",")
for key in LauncherLayer._profiles_named(part.strip(), profiles, every_profile)
})
if not readers:
# RetroDECK names its frontend and the core option names the core.
readers = setting_readers(
str(record.get("passes_to") or ""), profiles, every_profile
)
if emulator and emulator not in readers:
continue
folds = bool(record.get("case_insensitive") or config.get("case_insensitive_fs"))
base = str(config.get("base_destination") or "bios")
if record.get("kind") == "copies" and not launcher_directories(record, base):
continue
settings = name_settings(record, names)
def by_setting(name: str) -> bool:
"""Whether the setting the name is written to reads a held file.
A placeholder read from the setting takes the held dump the
catalogues name after the model the launcher's name spells.
"""
entries = [
entry
for key in readers
for entry in profiles[key].get("files") or []
if settings[name] and set_by(entry, settings[name])
]
return any(holder._entry_held(entry) for entry in entries) or any(
"<" in str(entry.get("name", ""))
and any(
name_match_size_ok(entry, found.get("size"))
for found in catalogued_model_files(name, db)
)
for entry in entries
)
by_model = {name for name in names if by_setting(name)}
if record.get("kind") == "names_searched" and any(
(found := holder.held(name)) and (found[1] or folds) for name in names
):
continue
reader = {
"emulator": str(record.get("emulator", "")),
"build": str(record.get("build", "")),
@@ -1180,9 +1241,19 @@ def launcher_reads_not_held(
"source_ref": str(record.get("source_ref", "")),
"profiles": readers,
}
# A name a reader's entry declares unsourceable is no gap: its
# row carries the reason and is counted apart.
unsourceable = {
str(n).rsplit("/", 1)[-1]: str(entry["unsourceable"])
for key in readers
for entry in profiles[key].get("files") or []
if entry.get("unsourceable")
for n in (entry.get("name"), *(entry.get("aliases") or []))
if n
}
for name in names:
found = holder.held(name)
if found and (found[1] or folds):
if found and (found[1] or folds) or name in by_model:
continue
key = name.casefold() if folds else name
row = by_key.get(key)
@@ -1195,6 +1266,9 @@ def launcher_reads_not_held(
}
if found:
row["held_as"] = found[0]
reason = unsourceable.get(name.rsplit("/", 1)[-1])
if reason and "unsourceable" not in row:
row["unsourceable"] = reason
if reader["emulator"] not in row["emulators"]:
row["emulators"].append(reader["emulator"])
if reader not in row["readers"]:
@@ -1204,7 +1278,13 @@ def launcher_reads_not_held(
def print_launcher_reads(rows: list[dict]) -> None:
"""The launcher reads gap category, after the profile gaps."""
"""The launcher reads gap category, after the profile gaps.
A name its reader declares unsourceable is listed apart, with the
reason: it is no file left to find.
"""
unsourceable = [r for r in rows if r.get("unsourceable")]
rows = [r for r in rows if not r.get("unsourceable")]
platforms = sorted({r["platform"] for r in rows})
noun = "platform" if len(platforms) == 1 else "platforms"
print(f"\nLauncher reads, not held: {len(rows)} names across {len(platforms)} {noun}")
@@ -1218,6 +1298,19 @@ def print_launcher_reads(rows: list[dict]) -> None:
f" {row['name']} [{', '.join(row['emulators'])}, {first['kind']}, "
f"{first['source_ref']}]{more}{casing}"
)
if not unsourceable:
return
platforms = sorted({r["platform"] for r in unsourceable})
noun = "platform" if len(platforms) == 1 else "platforms"
print(
f"\nLauncher reads, unsourceable: {len(unsourceable)} names "
f"across {len(platforms)} {noun}"
)
for row in unsourceable:
print(
f" {row['platform']}: {row['name']} [{', '.join(row['emulators'])}, "
f"{row['readers'][0]['source_ref']}]: {row['unsourceable']}"
)
def cross_reference(
@@ -1231,6 +1324,7 @@ def cross_reference(
folded_slots: frozenset[str] = frozenset(),
platform: str = "",
launcher_layer: LauncherLayer | None = None,
declared_contents: dict[str, set[str]] | None = None,
) -> dict:
"""Compare emulator profiles against platform declarations.
@@ -1268,6 +1362,7 @@ def cross_reference(
"data_names": data_names,
"all_declared": all_declared,
"folded_slots": folded_slots,
"declared_contents": declared_contents,
"bios_dir": db.get("bios_dir"),
"launcher_candidates": [] if launcher_layer else None,
}
@@ -1474,6 +1569,13 @@ def main() -> None:
all_declared=declared_names(args.platforms_dir, selected, db),
standalone_cores=standalone_cores,
folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
declared_contents=merged_platform_declared_contents(
[
load_platform_config(name, args.platforms_dir)
for name in selected or list_registered_platforms(args.platforms_dir)
],
db,
),
platform=config.get("_key", args.platform) if args.platform else "",
# What the platform's launchers open, as verify and the builder read it.
launcher_layer=LauncherLayer(
+205 -6
View File
@@ -65,6 +65,7 @@ class LauncherPlace:
anchored: bool
conditional: bool
source_ref: str
passes_to: str = ""
def fold(self, name: str) -> str:
return name.lower() if self.case_insensitive else name
@@ -87,6 +88,103 @@ class LauncherPlace:
)
def setting_tokens(passes_to: str) -> set[str]:
"""The words of a record's ``passes_to``, as an emulator setting is spelled.
"command line -romfile4p" hands the file to sdltrs's romfile4p option,
"settings.bml NeoGeoPocket/Firmware/BIOS.World" to an ares setting.
"""
return {
word.strip("()\"',;").lstrip("-")
for word in passes_to.split()
} - {""}
def suffixed_stem(name: str) -> str | None:
"""The file name a launcher name carries before an extension it appended.
ROCKNIX's Amiberry config passes kick34005.A500.rom, the name Amiberry
gives Kickstart 1.3 (amiberry_whdbooter.cpp:370) followed by ".rom". A
stem that is no file name itself (scph1001 of scph1001.bin) is none.
"""
base = name.rsplit("/", 1)[-1]
stem = base.rsplit(".", 1)[0]
return stem if "." in stem else None
_MODEL = re.compile(r"\b([A-Z]{2,5})-?(\d{3,6}[A-Z]?)\b")
def catalogued_model_files(name: str, db: dict) -> list[dict]:
"""Held files a dump catalogue describes as the model a launcher name spells.
RetroDECK sets LRPS2's pcsx2_bios option to scph10000.bin, a name no code
hashes: the SCPH-10000 is the model whose BIOS it means. Redump and TOSEC
describe two revisions for it. Primaries come before .variants copies,
then the Redump name, which carries the version (ps2-0100j before
ps2-0101j), then the No-Intro and TOSEC names.
"""
stem = name.rsplit("/", 1)[-1].rsplit(".", 1)[0].casefold()
if not re.fullmatch(r"[a-z]{2,5}\d{3,6}[a-z]?", stem):
return []
found = []
for record in db.get("files", {}).values():
catalogues = record.get("provenance") or {}
models = {
(a + b).casefold()
for entry in catalogues.values()
for a, b in _MODEL.findall(str(entry.get("description") or ""))
}
if stem in models:
names = [
str((catalogues.get(source) or {}).get("name") or "")
for source in ("redump", "no-intro", "tosec")
]
found.append(("/.variants/" in str(record.get("path", "")), names, record))
# The core accepts every revision of the model, and neither priority nor
# region separates two primaries. The pick must still be the same on
# every run, so it goes to the original revision: the Redump name carries
# the version, and ps2-0100j sorts before ps2-0101j.
return [record for *_key, record in sorted(found, key=lambda f: (f[0], f[1]))]
def name_settings(record: dict, names: list[str]) -> dict[str, str]:
"""The setting each name of a record is written to.
A record naming several files of one config spells passes_to as
describe_keys does (shell_launchers.py), "atari800.cfg ROM_OS_A_PAL,
ROM_5200_A, ...", one key per name and in the order of the names. When
the counts differ, each name keeps the whole description.
"""
passes_to = str(record.get("passes_to") or "")
head, _, rest = passes_to.partition(" ")
keys = [
key.strip()
for key in re.sub(r"\[[^\]]*\]", ",", rest).split(",")
if key.strip()
]
if len(names) > 1 and len(keys) == len(names) and all(" " not in k for k in keys):
return {name: f"{head} {key}" for name, key in zip(names, keys)}
return {name: passes_to for name in names}
def setting_readers(passes_to: str, profiles: dict, keys: Iterable[str]) -> list[str]:
"""The profiles with an entry read from the setting a launcher writes."""
if not passes_to:
return []
return sorted(
key for key in keys
if (profile := profiles.get(key)) is not None
and any(set_by(entry, passes_to) for entry in profile.get("files") or [])
)
def set_by(entry: dict, passes_to: str) -> bool:
"""Whether a launcher hands its file to the setting a profile entry is read from."""
key = str(entry.get("config_key") or "")
return bool(key) and key in setting_tokens(passes_to)
def _bios_relative(directory: str, base: str) -> str | None:
"""A launcher directory relative to the BIOS root, None when it is not one."""
directory = directory.strip().rstrip("/")
@@ -190,15 +288,21 @@ def launcher_places(config: dict) -> list[LauncherPlace]:
"conditional": bool(record.get("depends_on_option")),
"source_ref": str(record.get("source_ref", "")),
}
settings = name_settings(record, names)
if kind == "names_searched":
# Each directory in turn, every name in it.
depth = len(directories) if anchored else 1
ordered = tuple(paths_of(name)[i] for i in range(depth) for name in names)
places.append(LauncherPlace(paths=ordered, **common))
places.append(LauncherPlace(
paths=ordered, passes_to=str(record.get("passes_to") or ""), **common
))
continue
# A fixed name is the one file passed, and a core option offers
# each value as the file the user picks: one place per name.
places.extend(LauncherPlace(paths=paths_of(name), **common) for name in names)
places.extend(
LauncherPlace(paths=paths_of(name), passes_to=settings[name], **common)
for name in names
)
return places
@@ -551,6 +655,8 @@ class _Candidate:
by_content: bool
# Matched by the size the code checks, the entry naming no hash.
by_shape: bool = False
# Matched through the setting the launcher writes or the name it extends.
by_setting_or_stem: bool = False
class LauncherLayer:
@@ -580,12 +686,30 @@ class LauncherLayer:
self._by_name_folded.setdefault(name.lower(), []).extend(sha1s)
self._files = db.get("files", {})
self._owners: dict[tuple[str, bool], list[LauncherPlace]] = {}
self._unoffered: dict[tuple[str, bool], list[tuple[dict, dict]]] = {}
relevant = set(relevant)
for place in self.places:
standalone = place.build == "standalone"
for emulator in (e.strip() for e in place.emulator.split(",")):
for key in self._profiles_named(emulator, profiles, relevant):
self._owners.setdefault((key, standalone), []).append(place)
owners = {
key
for emulator in (e.strip() for e in place.emulator.split(","))
for key in self._profiles_named(emulator, profiles, relevant)
}
# RetroDECK names its frontend, retroarch, and the core option it
# writes, pcsx2_bios, names the core.
if not owners:
owners = set(setting_readers(place.passes_to, profiles, relevant))
for key in sorted(owners):
self._owners.setdefault((key, standalone), []).append(place)
# The report never offers a placeholder, which stands for every
# file a scan finds, nor an entry the platform's list settles
# (AmberELEC declares kick34005.A500): the launcher still reads
# its file under its own name.
self._unoffered.setdefault((key, standalone), [
(entry, _placeholder_report(key, profiles[key], entry))
for entry in profiles[key].get("files") or []
])
self._db = db
@staticmethod
def _profiles_named(name: str, profiles: dict, relevant: set[str]) -> list[str]:
@@ -622,6 +746,17 @@ class LauncherLayer:
names = own & basenames
if names:
return _Candidate(entry, names, False)
# RetroPie runs sdl2trs with -romfile4p $biosdir/level4p.rom: the
# file under that name is the one the romfile4p slot reads.
if set_by(entry, place.passes_to):
return _Candidate(entry, basenames, False, by_setting_or_stem=True)
if place.kind == "fixed_name":
stems = {
base for base in basenames
if (stem := suffixed_stem(base)) is not None and place.fold(stem) in own
}
if stems:
return _Candidate(entry, stems, False, by_setting_or_stem=True)
content = {
place.fold(path.rsplit("/", 1)[-1])
for path in place.paths
@@ -686,13 +821,26 @@ class LauncherLayer:
by_layout: dict[tuple[str, bool], list[tuple[dict, dict]]] = {}
for emulator, standalone, entry, report in candidates:
by_layout.setdefault((emulator, standalone), []).append((entry, report))
offered = {id(entry) for _e, _s, entry, _r in candidates}
for layout, members in self._unoffered.items():
for entry, report in members:
if id(entry) in offered:
continue
if "<" not in str(entry.get("name", "")):
report["in_repo"] = resolve_held(report) is not None
report["unoffered"] = True
by_layout.setdefault(layout, []).append((entry, report))
placed: set[str] = set()
for (emulator, standalone), members in sorted(by_layout.items()):
for place in self.places_for(emulator, standalone):
# An entry the report left out answers only through the
# setting or the extended name: by its own name it was left
# out for a reason (a region variant, a settled duplicate).
matched = [
(candidate, report)
for entry, report in members
if (candidate := self.match(place, entry)) is not None
and (candidate.by_setting_or_stem or not report.get("unoffered"))
]
# A size names one entry only when no other of the core's
# entries takes that size too.
@@ -745,7 +893,31 @@ class LauncherLayer:
):
return True
if not place.anchored:
continue
# A setting holds a name the core resolves in the directory
# of the entry read from it: LRPS2 joins pcsx2_bios under
# pcsx2/bios.
setting = [
(candidate, report) for candidate, report in matched
if set_by(candidate.entry, place.passes_to)
]
directory = next(
(
str(c.entry.get("path") or "").rpartition("/")[0]
for c, _report in setting
if "/" in str(c.entry.get("path") or "")
),
None,
)
if directory is None:
continue
path = f"{directory}/{name}"
if place.fold(path) in folded_reported:
return True
source = self._content_for(place, name, key, setting, resolve_held)
if source is None or place.fold(path) in placed:
continue
placed.add(place.fold(path))
return _launcher_entry(source[0], name, path, source[1], place)
# The emulator as this platform installs it brings the file: the
# place is filled, nothing is packed (ROCKNIX's xemu hdd image).
if any(key in candidate.names and report.get("bundled")
@@ -794,9 +966,36 @@ class LauncherLayer:
record = resolve_held(report)
if record is not None:
return report, record
for candidate, report in matched:
if key not in candidate.names or not set_by(candidate.entry, place.passes_to):
continue
for record in catalogued_model_files(name, self._db):
if name_match_size_ok(candidate.entry, record.get("size")):
return report, record
return None
def _placeholder_report(key: str, profile: dict, entry: dict) -> dict:
"""The report entry of an entry the gap report never builds."""
identity = {
field: entry[field]
for field in ("aliases", "md5", "sha1", "sha256", "crc32", "size", "min_size", "max_size")
if entry.get(field) is not None
}
return {
**identity,
"profile": key,
"emulator": profile.get("emulator", key),
"name": entry.get("name", ""),
"path": entry.get("path") or entry.get("name", ""),
"system": entry.get("system"),
"systems": profile.get("systems"),
"required": entry.get("required", False),
"category": entry.get("category"),
"in_repo": False,
}
def _launcher_entry(
report: dict, name: str, path: str, record: dict, place: LauncherPlace
) -> dict:
+13
View File
@@ -606,9 +606,15 @@ def declaration_settles(
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
declared_contents: Mapping[str, set[str]] | None = None,
) -> bool:
"""Whether a platform declaration meets a core's requirement.
An entry the core finds by content (``found_by: content`` on the entry)
is met by any declaration of a file it accepts by hash, in the folder it
scans: atari800 takes RetroDECK's ROM_BASIC_A by its CRC32, so its own
label for that ROM is not one more file to ship.
A platform declaring the name the core reads has met the requirement
where the declaration fills the entry's own slot, whose content the slot
arbitration decides, or elsewhere at a size the core accepts: the builder
@@ -630,6 +636,13 @@ def declaration_settles(
return True
slot = sanitize_pack_path(dest or fname).lower()
directory = slot if dest.endswith("/") else slot.rpartition("/")[0]
if declared_contents and file_entry.get("found_by") == "content":
for field in ("md5", "sha1", "crc32"):
values = file_entry.get(field) or []
for value in values if isinstance(values, list) else str(values).split(","):
folders = declared_contents.get(f"{field}:{str(value).strip().lower()}")
if folders and directory in folders:
return True
if not dest.endswith("/") and slot in folded_slots:
return True
for name in (fname, *file_entry.get("aliases", [])):
+14 -5
View File
@@ -60,6 +60,7 @@ from common import (
compute_hashes,
expand_directory_entries,
platform_declarations,
platform_declared_contents,
group_identical_platforms,
list_emulator_profiles,
list_system_ids,
@@ -454,6 +455,7 @@ def _candidate_verdict(
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
declared_contents: Mapping[str, set[str]] | None = None,
) -> str:
"""Whether a profile entry can be a gap, and whether it is settled.
@@ -469,7 +471,7 @@ def _candidate_verdict(
if outside_gap_scope(file_entry, is_standalone):
return "skip"
if not include_all and declaration_settles(
file_entry, fname, declared_names, dest, folded_slots
file_entry, fname, declared_names, dest, folded_slots, declared_contents
):
return "settled"
return "keep"
@@ -503,6 +505,7 @@ def find_undeclared_files(
if declared_names is None:
declared_names = platform_declarations(config, db)
folded_slots = case_folded_slots(config, declared_names)
declared_contents = platform_declared_contents(config, db)
# Whether the builder drops a file whose local copy contradicts its
# declared hash, which decides if such a copy counts as held here.
@@ -608,6 +611,7 @@ def find_undeclared_files(
f, fname, is_standalone, include_all,
{} if save_root or f.get("root") == "roms" else declared_names,
dest, folded_slots,
None if save_root or f.get("root") == "roms" else declared_contents,
)
if verdict == "settled":
seen_files.add(seen_key)
@@ -1986,13 +1990,18 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
def _print_launcher_reads(result: dict, verbose: bool) -> None:
"""Names the platform's launch code reads that the collection lacks."""
rows = result.get("launcher_reads_not_held") or []
if not rows:
return
every = result.get("launcher_reads_not_held") or []
rows = [r for r in every if not r.get("unsourceable")]
unsourceable = [r for r in every if r.get("unsourceable")]
listed = "" if verbose else " (--verbose lists them)"
print(f" Launcher reads, not held: {len(rows)}{listed}")
if rows:
print(f" Launcher reads, not held: {len(rows)}{listed}")
if unsourceable:
print(f" Launcher reads, unsourceable: {len(unsourceable)}{listed}")
if not verbose:
return
for row in unsourceable:
print(f" {row['name']} [{', '.join(row['emulators'])}]: {row['unsourceable']}")
for row in rows:
kinds = ", ".join(sorted({r["kind"] for r in row["readers"]}))
casing = f", held as {row['held_as']}" if row.get("held_as") else ""