kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-27 14:17:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: file mame software lists and floppy samples

Browse Source 
The -listxml names the software lists each machine and slot card
declares and the floppy drives it can instantiate. --devices now writes
one entry per hash/<list>.xml the release ships under the systems of its
placed machines (111 lists, 666 stay with hash/), and files each floppy
sample under the systems whose drives select its form factor
(floppy.cpp:1428-1473, 1759-1767).

Machines a registered platform declares under one of the profile's
systems seed the walk too: bbcb, cdimono1, cdimono2, dragon32, ep64 and
ep128. Unplaced device sets go from 903 to 861.
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 12:24:37 +02:00
1 parent 7ac96f2b39
commit 2d74524d59
9 files changed
+1959 -26
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/scraper/_hash_merge.py
+134 -5
View File
@@ -21,6 +21,7 @@ from common import PROFILE_BACKUP_SUFFIX, write_text_atomic, yaml_load
_MAME_RELEASE_RE = re.compile(r"^0\.\d+")
_MAMEDEV = "https://github.com/mamedev/mame"
DEVICE_SECTION = " # --- Device ROM sets (generated from MAME -listxml) ---"
SOFTLIST_SECTION = " # --- Software lists (generated from MAME -listxml) ---"
_FILES_END = re.compile(r"^\S", re.M)
@@ -553,9 +554,9 @@ def _append_new_entries(text: str, files: list[dict], original: str) -> str:
lines.append(_format_contents(fe["contents"]))
if lines:
# New BIOS root sets go before the generated device section, which
# runs to the end of the files list.
at = _device_section_start(text)
# New BIOS root sets go before the generated sections, which run to
# the end of the files list.
at = generated_start(text)
if at is None:
text = text.rstrip("\n") + "\n" + "\n".join(lines) + "\n"
else:
@@ -570,6 +571,18 @@ def _device_section_start(text: str) -> int | None:
return match.start() if match else None
def _softlist_section_start(text: str) -> int | None:
"""Offset of the generated software list section header, if any."""
match = re.search(rf"^{re.escape(SOFTLIST_SECTION)}$", text, re.M)
return match.start() if match else None
def generated_start(text: str) -> int | None:
"""Offset of the first generated section: what a reader wrote ends there."""
found = [at for at in (_softlist_section_start(text), _device_section_start(text)) if at is not None]
return min(found) if found else None
def _device_contents(roms: list[dict], old: list[dict]) -> list[dict]:
"""Contents of a device set, keeping the member descriptions a reader wrote."""
written = {str(c.get("name")): c.get("description") for c in old or []}
@@ -641,7 +654,11 @@ def _set_section_systems(text: str, systems: dict[str, list[str]]) -> str:
start = _device_section_start(text)
if start is None:
return text
head, section = text[:start], text[start:]
return text[:start] + _rewrite_systems(text[start:], systems)
def _rewrite_systems(section: str, systems: dict[str, list[str]]) -> str:
"""Rewrite the system fields of the named entries of one stretch of text."""
out: list[str] = []
current = ""
in_list = False
@@ -661,7 +678,7 @@ def _set_section_systems(text: str, systems: dict[str, list[str]]) -> str:
out.append(line)
if current in systems and re.match(r"^ category:", line):
out.extend(_system_lines(systems[current]))
return head + "\n".join(out)
return "\n".join(out)
def _format_device_entry(name: str, device: dict, role: str) -> str:
@@ -762,6 +779,118 @@ def merge_mame_devices(
return report
def _softlist_section(profile: dict, lists: dict[str, list[str]], role: str) -> str:
"""The generated software list section, one entry per list and tree.
A list takes the fields of each hash/ tree the profile delivers (a
bundled tree ships with the emulator, not in a pack) and the path of
that tree. Members the section names leave the tree's own entry.
"""
trees = [
entry for entry in profile.get("files") or []
if entry.get("type") == "directory"
and str(entry.get("name")) == "hash/"
and not entry.get("bundled")
]
blocks: list[str] = []
for tree in trees:
base = str(tree.get("path") or tree["name"])
for name, systems in sorted(lists.items()):
lines = [
f" - name: {_yaml_scalar(name + '.xml')}",
f" path: {_yaml_scalar(base + name + '.xml')}",
]
if tree.get("mode"):
lines.append(f" mode: {tree['mode']}")
lines += [
" category: game_data",
" required: false",
*_system_lines(systems),
]
blocks.append("\n".join(lines))
if not blocks:
return ""
lead = [
SOFTLIST_SECTION,
" # software_list_device::parse opens <list>.xml for each software list a",
" # machine declares. Each list below is filed under the systems whose",
" # machines declare it, a list no such machine declares stays with its",
" # hash/ tree.",
f" # ref: {role}src/emu/softlist_dev.cpp:281-313,",
f" # {role}src/frontend/mame/infoxml.cpp:2184-2198",
]
return "\n".join(lead) + "\n\n" + "\n\n".join(blocks) + "\n"
def merge_mame_support(
profile_path: str,
lists: dict[str, list[str]],
samples: dict[str, list[str]],
write: bool = False,
) -> dict[str, list[str]]:
"""File the software lists and the floppy samples of a MAME profile.
*lists* maps a software list to the systems whose machines declare it,
for the lists the release ships. They are written as the generated
software list section, ahead of the device section, and a refresh
rewrites the whole section. *samples* maps a floppy sample name to the
systems whose drives load it: each written entry named <sample>.wav
under a samples/floppy/ path gets them, and one no system loads gets
none.
"""
path = Path(profile_path)
text = path.read_text(encoding="utf-8")
profile = yaml_load(text) or {}
role = mame_ref_role(profile)
original = text
start = _softlist_section_start(text)
if start is not None:
device = _device_section_start(text)
if device is not None and device > start:
end = device
else:
end_match = _FILES_END.search(text, text.index("\n", start) + 1)
end = end_match.start() if end_match else len(text)
text = text[:start] + text[end:]
section = _softlist_section(profile, lists, role)
head_end = generated_start(text)
head = text if head_end is None else text[:head_end]
written = yaml_load(head) or {}
sample_systems: dict[str, list[str]] = {}
for entry in written.get("files") or []:
name = str(entry.get("name") or "")
if "samples/floppy/" not in str(entry.get("path") or "") or not name.endswith(".wav"):
continue
wanted = samples.get(name[:-4], [])
if _entry_systems(entry) != wanted:
sample_systems[name] = wanted
head = _rewrite_systems(head, sample_systems) if sample_systems else head
rest = "" if head_end is None else text[head_end:]
if section and not rest:
files_at = re.search(r"^files:[ \t]*$", head, re.M)
if files_at is None or _FILES_END.search(head, files_at.end()):
raise ValueError(
"files: must be the last top-level key to hold a software list section"
)
patched = head.rstrip("\n") + "\n"
if section:
patched += "\n" + section
if rest:
patched += "\n" + rest
if write and patched != original:
shutil.copy2(path, path.with_suffix(PROFILE_BACKUP_SUFFIX))
write_text_atomic(str(path), patched)
return {
"lists": sorted(lists),
"samples": sorted(sample_systems),
"changed": [str(path)] if patched != original else [],
}
def _add_source_refs(text: str, refs: dict[str, str]) -> str:
"""Give each named entry a source_ref line, right after its category."""
if not refs:
scripts/scraper/mame_devices.py
+175 -11
View File
@@ -10,7 +10,10 @@ at the release tag. A set whose block cannot be told apart is left without
lines and reported.
The same document says which machines load each set, so a set is filed under
the systems of those machines (attribute_systems).
the systems of those machines (attribute_systems). It also names the software
lists each machine declares and the floppy drives it can instantiate, which
files hash/<list>.xml (attribute_softlists) and the floppy sound samples
(floppy_sample_systems) the same way.
"""
from __future__ import annotations
@@ -19,7 +22,7 @@ import html
import io
import re
import zipfile
from collections.abc import Callable, Iterator
from collections.abc import Callable, Collection, Iterable, Iterator
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path
@@ -27,7 +30,7 @@ from typing import NamedTuple
from common import yaml_load
from ._hash_merge import _device_section_start
from ._hash_merge import generated_start
from .mame_parser import _collect_rom_macros, _expand_macros, _parse_rom_entries
LISTXML_CHUNK = 8 * 1024 * 1024
@@ -40,6 +43,7 @@ _BIOSSET = re.compile(rb"<biosset\b([^>]*)>")
_ROM = re.compile(rb"<rom\b([^>]*)>")
_DEVICE_REF = re.compile(rb'<device_ref\b[^>]*\bname="([^"]+)"')
_SLOT_OPTION = re.compile(rb'<slotoption\b[^>]*\bdevname="([^"]+)"')
_SOFTWARE_LIST = re.compile(rb'<softwarelist\b[^>]*\bname="([^"]+)"')
_ROM_START = re.compile(r"ROM_START\s*\(\s*(\w+)\s*\)")
_ROM_END = re.compile(r"\bROM_END\b")
@@ -112,12 +116,15 @@ class MachineLinks(NamedTuple):
*romof* is the parent the element names. *uses* are the devices it can
instantiate: every subdevice of its default configuration, each device a
selectable slot option adds, and for a device its parent ROM device,
whose set the device's ROMs are also searched in.
whose set the device's ROMs are also searched in. *softlists* are the
software lists the element declares, *sourcefile* the file defining it.
"""
romof: str
isdevice: bool
uses: tuple[str, ...]
softlists: tuple[str, ...] = ()
sourcefile: str = ""
class Release(NamedTuple):
@@ -134,8 +141,9 @@ def scan_release(stream: io.BufferedIOBase) -> Release:
driver's parent (src/frontend/mame/infoxml.cpp:761-765) or a device's
parent ROM device (856-858), device_ref every subdevice of the default
configuration (786, 1010-1015), slotoption each selectable card of a
slot (2129-2172). device_t::searchpath puts the parent ROM device after
the device itself (src/emu/device.cpp:140-152).
slot (2129-2172), softwarelist each software list device of the
configuration (2184-2198). device_t::searchpath puts the parent ROM
device after the device itself (src/emu/device.cpp:140-152).
"""
tag = ""
devices: dict[str, dict] = {}
@@ -162,7 +170,11 @@ def scan_release(stream: io.BufferedIOBase) -> Release:
if isdevice and romof:
uses.add(romof)
uses.discard(name)
machines[name] = MachineLinks(romof, isdevice, tuple(sorted(uses)))
softlists = {found.decode() for found in _SOFTWARE_LIST.findall(body)}
machines[name] = MachineLinks(
romof, isdevice, tuple(sorted(uses)), tuple(sorted(softlists)),
attrs.get("sourcefile", ""),
)
if isdevice:
roms = _device_roms(body)
if roms:
@@ -187,13 +199,14 @@ def scan_device_sets(stream: io.BufferedIOBase) -> tuple[str, dict[str, dict]]:
def profile_seeds(text: str) -> dict[str, str]:
"""Set name to system, for the bios_zip entries above the device section.
"""Set name to system, for the bios_zip entries before the generated ones.
These are the entries a reader wrote, the seeds attribute_systems starts
from. The generated section is left out: its systems derive from these,
and reading them back would carry a previous release's answer forward.
from. The generated sections are left out: their systems derive from
these, and reading them back would carry a previous release's answer
forward.
"""
start = _device_section_start(text)
start = generated_start(text)
written = yaml_load(text if start is None else text[:start]) or {}
seeds: dict[str, str] = {}
for entry in written.get("files") or []:
@@ -259,6 +272,157 @@ def attribute_systems(
return {name: sorted(found) for name, found in systems.items()}
def platform_seeds(
stated: Iterable[tuple[str, str]],
machines: dict[str, MachineLinks],
systems: Collection[str],
written: dict[str, str],
) -> tuple[dict[str, str], dict[str, list[str]]]:
"""Seeds a platform file states, and the sets it states two ways.
*stated* are (archive name, system) pairs a platform declares. A pair
seeds a set when the archive is a machine of the release, not a device,
the system is one the profile lists, and the profile has not written
the set itself. A set stated under two of the profile's systems seeds
nothing and is returned with them.
"""
found: dict[str, set[str]] = {}
listed = set(systems)
for archive, system in stated:
if not archive.endswith(".zip") or system not in listed:
continue
name = archive[:-4]
links = machines.get(name)
if name in written or links is None or links.isdevice:
continue
found.setdefault(name, set()).add(system)
seeds = {name: next(iter(s)) for name, s in found.items() if len(s) == 1}
conflicts = {name: sorted(s) for name, s in found.items() if len(s) > 1}
return seeds, conflicts
def attribute_softlists(
machines: dict[str, MachineLinks], placed: dict[str, list[str]]
) -> dict[str, list[str]]:
"""Systems whose machines declare each software list.
*placed* is what attribute_systems returns. A list goes to the systems
of every element declaring it: a machine writes the lists of its whole
configuration and a slot card its own (infoxml.cpp:2184-2198).
software_list_device::parse opens <list>.xml from the hash path
(src/emu/softlist_dev.cpp:281-313).
"""
lists: dict[str, set[str]] = {}
for name, systems in placed.items():
links = machines.get(name)
for listed in links.softlists if links else ():
lists.setdefault(listed, set()).update(systems)
return {name: sorted(found) for name, found in lists.items()}
# floppy_image_device adds its sound device under this shortname
# (src/devices/imagedev/floppy.cpp:50, 2076-2082).
FLOPPY_SOUND = "flopsnd"
_FORM_FACTOR = re.compile(r"\bm_form_factor\s*=\s*floppy_image::(FF_\w+)")
_SAMPLE_FORM_FACTOR = re.compile(
r"\bm_default_samples\.set_form_factor\s*\(\s*floppy_image::(FF_\w+)"
r"|\bm_default_samples\.add_\w+_sample\s*\(\s*\"([^\"]+)\""
)
def default_floppy_samples(text: str) -> dict[str, str]:
"""Each sample of floppy_sound_device's default list, to its form factor.
The constructor sets a form factor, then adds the samples recorded for
it (src/devices/imagedev/floppy.cpp:1656-1686).
"""
body = re.search(
r"floppy_sound_device::floppy_sound_device\s*\([^)]*\)[^{]*\{(.*?)\n\}",
blank_comments(text),
re.S,
)
samples: dict[str, str] = {}
current = ""
for match in _SAMPLE_FORM_FACTOR.finditer(body.group(1) if body else ""):
if match.group(1):
current = match.group(1)
elif current:
samples[match.group(2)] = current
return samples
def drive_form_factor(text: str, shortname: str) -> str:
"""Form factor a floppy drive type sets in setup_characteristics, or ""."""
text = blank_comments(text)
declaration = re.search(
r"DEFINE_DEVICE_TYPE\w*\s*\(\s*\w+\s*,\s*(\w+)\s*,\s*\"" + re.escape(shortname) + '"',
text,
)
if declaration is None:
return ""
body = re.search(
rf"\b{re.escape(declaration.group(1))}::setup_characteristics\s*\(\s*\)\s*\{{(.*?)\n\}}",
text,
re.S,
)
found = _FORM_FACTOR.search(body.group(1)) if body else None
return found.group(1) if found else ""
def selected_form_factor(form_factor: str, available: Collection[str]) -> str:
"""The list floppy_sound_samples::select takes for a drive's form factor.
3" falls back to 3.5", anything else to 5.25", and a form factor no list
answers takes none (src/devices/imagedev/floppy.cpp:1428-1473).
"""
while form_factor not in available:
if form_factor == "FF_3":
form_factor = "FF_35"
elif form_factor != "FF_525":
form_factor = "FF_525"
else:
return ""
return form_factor
def floppy_drives(machines: dict[str, MachineLinks]) -> list[str]:
"""Device types that instantiate the floppy sound device."""
return sorted(
name for name, links in machines.items()
if links.isdevice and FLOPPY_SOUND in links.uses
)
def floppy_sample_systems(
machines: dict[str, MachineLinks],
placed: dict[str, list[str]],
samples: dict[str, str],
form_factors: dict[str, str],
) -> dict[str, list[str]]:
"""Systems whose machines load each floppy sound sample.
Every drive starts a floppy sound device, which loads the default list
of the form factor its drive selects (floppy.cpp:1709-1738,
1759-1767). *samples* maps a sample to its form factor
(default_floppy_samples), *form_factors* a drive to its own
(drive_form_factor). A drive whose form factor is unknown is taken to
load every list.
"""
available = set(samples.values())
found: dict[str, set[str]] = {}
for drive in floppy_drives(machines):
systems = placed.get(drive)
if not systems:
continue
own = form_factors.get(drive, "")
chosen = {selected_form_factor(own, available)} if own else available
for sample, form_factor in samples.items():
if form_factor in chosen:
found.setdefault(sample, set()).update(systems)
return {name: sorted(s) for name, s in found.items()}
def source_path(sourcefile: str) -> str:
"""Repository path of a listxml sourcefile.
scripts/scraper/mame_hash_scraper.py
+76 -6
View File
@@ -23,7 +23,14 @@ from typing import Any
import yaml
from common import github_token, source_yaml_paths, write_text_atomic, yaml_load
from common import (
github_token,
list_registered_platforms,
load_platform_config,
source_yaml_paths,
write_text_atomic,
yaml_load,
)
import upstream
@@ -33,13 +40,21 @@ from ._hash_merge import (
compute_diff,
merge_mame_devices,
merge_mame_profile,
merge_mame_support,
)
from .mame_devices import (
attribute_softlists,
attribute_systems,
default_floppy_samples,
drive_form_factor,
floppy_drives,
floppy_sample_systems,
locate_sources,
open_listxml,
platform_seeds,
profile_seeds,
scan_release,
source_path,
)
from .mame_parser import parse_mame_source_tree
@@ -51,6 +66,8 @@ _CACHE_PATH = _ROOT / "data" / "mame-hashes.json"
# release by a second run while the first was still parsing it.
_CLONE_DIR = _ROOT / "tmp" / f"mame-{os.getpid()}"
_EMULATORS_DIR = _ROOT / "emulators"
_PLATFORMS_DIR = _ROOT / "platforms"
_FLOPPY_SOURCE = "src/devices/imagedev/floppy.cpp"
_REPO_URL = "https://github.com/mamedev/mame.git"
_STALE_HOURS = 24
@@ -474,6 +491,17 @@ def _write_release(args: argparse.Namespace) -> None:
print(f"{args.profile}: {len(profile['files'])} BIOS root sets at {args.release}")
def _platform_archives() -> list[tuple[str, str]]:
"""(file name, system) for every file a registered platform declares."""
stated: list[tuple[str, str]] = []
for name in list_registered_platforms(str(_PLATFORMS_DIR), include_archived=True):
config = load_platform_config(name, str(_PLATFORMS_DIR))
for system, data in (config.get("systems") or {}).items():
for entry in (data or {}).get("files") or []:
stated.append((str(entry.get("name") or ""), str(system)))
return stated
def _merge_devices(args: argparse.Namespace) -> None:
"""Document the device ROM sets of the release mame.yml is pinned on.
@@ -498,15 +526,39 @@ def _merge_devices(args: argparse.Namespace) -> None:
f"{tag} is {commit}, mame.yml upstream_commit is "
f"{profile.get('upstream_commit')}"
)
unplaced = locate_sources(
devices,
lambda path: upstream.fetch_file(repo, commit, path, upstream.CACHE_DIR),
def fetch(path: str) -> list[str] | None:
return upstream.fetch_file(repo, commit, path, upstream.CACHE_DIR)
unplaced = locate_sources(devices, fetch)
written = profile_seeds(text)
stated, conflicts = platform_seeds(
_platform_archives(), machines, profile.get("systems") or [], written
)
systems = attribute_systems(machines, profile_seeds(text))
systems = attribute_systems(machines, {**stated, **written})
for set_name, device in devices.items():
device["systems"] = systems.get(set_name, [])
report = merge_mame_devices(str(profile_path), devices, write=not args.dry_run)
shipped = upstream.list_directory(repo, commit, "hash", upstream.CACHE_DIR)
if shipped is None:
raise RuntimeError(f"hash/ of {repo.slug} at {commit} could not be listed")
present = {name[:-4] for name in shipped if name.endswith(".xml")}
declared = attribute_softlists(machines, systems)
lists = {name: found for name, found in declared.items() if name in present}
floppy = fetch(_FLOPPY_SOURCE)
if floppy is None:
raise RuntimeError(f"{_FLOPPY_SOURCE} is absent at {commit}")
samples = default_floppy_samples("\n".join(floppy))
form_factors: dict[str, str] = {}
for drive in floppy_drives(machines):
lines = fetch(source_path(machines[drive].sourcefile))
form_factors[drive] = drive_form_factor("\n".join(lines or []), drive)
sample_systems = floppy_sample_systems(machines, systems, samples, form_factors)
support = merge_mame_support(
str(profile_path), lists, sample_systems, write=not args.dry_run
)
print(f"mame-devices: {len(devices)} device ROM sets in the {tag} listxml")
print(f" {profile_path.name}:")
print(f" + {len(report['added'])} added")
@@ -518,9 +570,27 @@ def _merge_devices(args: argparse.Namespace) -> None:
print(
f" systems: {by_count[1]} sets under one system, "
f"{sum(n for k, n in by_count.items() if k > 1)} under several, "
f"{by_count[0]} loaded by no machine a profile entry places "
f"{by_count[0]} loaded by no placed machine "
"(they serve every system of the profile)"
)
print(
f" seeds: {len(written)} written in {profile_path.name}, "
f"{len(stated)} from platform files"
)
for set_name, found in sorted(conflicts.items()):
print(f" {set_name}.zip stated under {', '.join(found)}, left unseeded")
print(
f" software lists: {len(lists)} filed under their systems, "
f"{len(present) - len(lists)} declared by no placed machine stay with hash/, "
f"{len(set(declared) - present)} placed lists absent from hash/"
)
unread = sorted(drive for drive, found in form_factors.items() if not found)
print(
f" floppy samples: {len(sample_systems)} of {len(samples)} filed, "
f"{len(support['samples'])} entries rewritten"
)
for drive in unread:
print(f" {drive}: no form factor read, taken to load every list")
if unplaced:
print(f" {len(unplaced)} without source lines:")
for set_name, reason in sorted(unplaced.items()):