diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index 3b35f59d..7607ca3e 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -1,9 +1,10 @@ """Scraper for Batocera per-board emulator availability. Sources (batocera-linux/batocera.linux): - - configs/batocera-*.board — board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_* - - package/batocera/core/batocera-system/Config.in — flag-to-package mapping - - es_systems.yml — emulator-to-requireAnyOf flag mapping + - configs/batocera-*.board -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_* + - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines + - package/batocera/emulationstation/batocera-es-system/es_systems.yml + -- emulator requireAnyOf flag mapping """ from __future__ import annotations @@ -24,12 +25,9 @@ PLATFORM_NAME = "batocera" GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents" RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master" -CONFIG_IN_URL = ( - f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in" -) +CONFIG_IN_URL = f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in" ES_SYSTEMS_URL = ( - f"{RAW_BASE}/package/batocera/emulationstation/batocera-emulationstation/" - "es_systems.yml" + f"{RAW_BASE}/package/batocera/emulationstation/batocera-es-system/es_systems.yml" ) _HEADERS = { @@ -38,19 +36,24 @@ _HEADERS = { } _TARGET_FLAG_RE = re.compile(r'^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y', re.MULTILINE) -_REQUIRE_ANYOF_RE = re.compile( - r'requireAnyOf\s*:\s*\[([^\]]+)\]', re.MULTILINE + +# Matches: select BR2_PACKAGE_FOO (optional: if CONDITION) +# Condition may span multiple lines (backslash continuation) +_SELECT_RE = re.compile( + r'^\s+select\s+(BR2_PACKAGE_\w+)' # package being selected + r'(?:\s+if\s+((?:[^\n]|\\\n)+?))?' # optional "if CONDITION" (may continue with \) + r'(?:\s*#[^\n]*)?$', # optional trailing comment + re.MULTILINE, ) - -def _arch_from_flag(flag: str) -> str: - """Guess architecture from board flag name.""" - low = flag.lower() - if "x86_64" in low or "x86-64" in low: - return "x86_64" - if "x86" in low and "64" not in low: - return "x86" - return "aarch64" +# Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif" +_META_BLOCK_RE = re.compile( + r'^if\s+((?:[^\n]|\\\n)+?)\n' # condition (may span lines via \) + r'(?:.*?\n)*?' # optional lines before the config + r'\s+config\s+(BR2_PACKAGE_\w+)' # derived flag name + r'.*?^endif', # end of block + re.MULTILINE | re.DOTALL, +) def _fetch(url: str, headers: dict | None = None) -> str | None: @@ -75,53 +78,175 @@ def _fetch_json(url: str) -> list | dict | None: return None -def _parse_config_in(text: str) -> dict[str, list[str]]: - """Parse Config.in: map BR2_PACKAGE_BATOCERA_TARGET_* flags to packages.""" - flag_to_packages: dict[str, list[str]] = {} - # Find blocks: if BR2_PACKAGE_BATOCERA_TARGET_X ... select BR2_PACKAGE_Y - block_re = re.compile( - r'if\s+(BR2_PACKAGE_BATOCERA_TARGET_\w+)(.*?)endif', - re.DOTALL, - ) - select_re = re.compile(r'select\s+(BR2_PACKAGE_\w+)') - for m in block_re.finditer(text): - flag = m.group(1) - block = m.group(2) - packages = select_re.findall(block) - flag_to_packages.setdefault(flag, []).extend(packages) - return flag_to_packages +def _normalise_condition(raw: str) -> str: + """Strip backslash-continuations and collapse whitespace.""" + return re.sub(r'\\\n\s*', ' ', raw).strip() + + +def _tokenise(condition: str) -> list[str]: + """Split a Kconfig condition into tokens: flags, !, &&, ||, (, ).""" + token_re = re.compile(r'&&|\|\||!|\(|\)|BR2_\w+|"[^"]*"') + return token_re.findall(condition) + + +def _check_condition(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: + """Recursive descent check of a Kconfig boolean expression.""" + return _check_or(tokens, pos, active) + + +def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: + left, pos = _check_and(tokens, pos, active) + while pos < len(tokens) and tokens[pos] == '||': + pos += 1 + right, pos = _check_and(tokens, pos, active) + left = left or right + return left, pos + + +def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: + left, pos = _check_not(tokens, pos, active) + while pos < len(tokens) and tokens[pos] == '&&': + pos += 1 + right, pos = _check_not(tokens, pos, active) + left = left and right + return left, pos + + +def _check_not(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: + if pos < len(tokens) and tokens[pos] == '!': + pos += 1 + val, pos = _check_atom(tokens, pos, active) + return not val, pos + return _check_atom(tokens, pos, active) + + +def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: + if pos >= len(tokens): + return True, pos + tok = tokens[pos] + if tok == '(': + pos += 1 + val, pos = _check_or(tokens, pos, active) + if pos < len(tokens) and tokens[pos] == ')': + pos += 1 + return val, pos + if tok.startswith('BR2_'): + pos += 1 + return tok in active, pos + if tok.startswith('"'): + pos += 1 + return True, pos + # Unknown token — treat as true to avoid false negatives + pos += 1 + return True, pos + + +def _condition_holds(condition: str, active: frozenset[str]) -> bool: + """Return True if a Kconfig boolean condition holds for the given active flags.""" + if not condition: + return True + norm = _normalise_condition(condition) + tokens = _tokenise(norm) + if not tokens: + return True + try: + result, _ = _check_condition(tokens, 0, active) + return result + except Exception: + return True # conservative: include on parse failure + + +def _parse_meta_flags(text: str) -> list[tuple[str, str]]: + """Return [(derived_flag, condition_str)] from top-level if/endif blocks. + + These define derived flags like BR2_PACKAGE_BATOCERA_TARGET_X86_64_ANY, + BR2_PACKAGE_BATOCERA_GLES3, etc. + """ + results: list[tuple[str, str]] = [] + for m in _META_BLOCK_RE.finditer(text): + cond = _normalise_condition(m.group(1)) + flag = m.group(2) + results.append((flag, cond)) + return results + + +def _expand_flags(primary_flag: str, meta_rules: list[tuple[str, str]]) -> frozenset[str]: + """Given a board's primary flag, expand to all active derived flags. + + Iterates until stable (handles chained derivations like X86_64_ANY -> X86_ANY). + """ + active: set[str] = {primary_flag} + changed = True + while changed: + changed = False + for derived, cond in meta_rules: + if derived not in active and _condition_holds(cond, frozenset(active)): + active.add(derived) + changed = True + return frozenset(active) + + +def _parse_selects(text: str) -> list[tuple[str, str]]: + """Parse all 'select PACKAGE [if CONDITION]' lines from Config.in. + + Returns [(package, condition)] where condition is '' if unconditional. + """ + results: list[tuple[str, str]] = [] + for m in _SELECT_RE.finditer(text): + pkg = m.group(1) + cond = _normalise_condition(m.group(2) or '') + results.append((pkg, cond)) + return results def _parse_es_systems(text: str) -> dict[str, list[str]]: - """Parse es_systems.yml: map emulator name to list of requireAnyOf flags.""" + """Parse es_systems.yml: map BR2_PACKAGE_* flag -> list of emulator names. + + The file is a dict keyed by system name. Each system has: + emulators: + : + : {requireAnyOf: [BR2_PACKAGE_FOO]} + """ try: data = yaml.safe_load(text) except yaml.YAMLError: return {} - emulator_flags: dict[str, list[str]] = {} if not isinstance(data, dict): - return emulator_flags + return {} - systems = data.get("systems", data) if "systems" in data else data - if not isinstance(systems, list): - # Could be a dict - systems = list(systems.values()) if isinstance(systems, dict) else [] + package_to_emulators: dict[str, list[str]] = {} - for system in systems: - if not isinstance(system, dict): + for _system_name, system_data in data.items(): + if not isinstance(system_data, dict): continue - for emulator_entry in system.get("emulators", []): - if not isinstance(emulator_entry, dict): + emulators = system_data.get("emulators") + if not isinstance(emulators, dict): + continue + for _group_name, group_data in emulators.items(): + if not isinstance(group_data, dict): continue - for emu_name, emu_data in emulator_entry.items(): - if not isinstance(emu_data, dict): + for core_name, core_data in group_data.items(): + if not isinstance(core_data, dict): continue - require = emu_data.get("requireAnyOf", []) - if isinstance(require, list): - emulator_flags.setdefault(emu_name, []).extend(require) + require = core_data.get("requireAnyOf", []) + if not isinstance(require, list): + continue + for pkg_flag in require: + if isinstance(pkg_flag, str): + package_to_emulators.setdefault(pkg_flag, []).append(core_name) - return emulator_flags + return package_to_emulators + + +def _arch_from_flag(flag: str) -> str: + """Guess architecture from board flag name.""" + low = flag.lower() + if "x86_64" in low or "zen3" in low: + return "x86_64" + if "x86" in low: + return "x86" + return "aarch64" class Scraper(BaseTargetScraper): @@ -159,44 +284,55 @@ class Scraper(BaseTargetScraper): print(" warning: no boards found", file=sys.stderr) print(" fetching Config.in...", file=sys.stderr) - config_in_text = _fetch(CONFIG_IN_URL) - flag_to_packages: dict[str, list[str]] = {} - if config_in_text: - flag_to_packages = _parse_config_in(config_in_text) + config_in_text = _fetch(CONFIG_IN_URL) or "" + + meta_rules = _parse_meta_flags(config_in_text) + selects = _parse_selects(config_in_text) + print( + f" parsed {len(meta_rules)} meta-flag rules, {len(selects)} select lines", + file=sys.stderr, + ) print(" fetching es_systems.yml...", file=sys.stderr) - es_text = _fetch(ES_SYSTEMS_URL) - emulator_flags: dict[str, list[str]] = {} - if es_text: - emulator_flags = _parse_es_systems(es_text) - - # Build reverse index: package -> emulators - package_to_emulators: dict[str, list[str]] = {} - for emu, flags in emulator_flags.items(): - for flag in flags: - package_to_emulators.setdefault(flag, []).append(emu) + es_text = _fetch(ES_SYSTEMS_URL) or "" + package_to_emulators = _parse_es_systems(es_text) + print( + f" parsed {len(package_to_emulators)} package->emulator mappings", + file=sys.stderr, + ) targets: dict[str, dict] = {} for board_name in sorted(boards): target_key = board_name.removeprefix("batocera-").removesuffix(".board") print(f" processing {target_key}...", file=sys.stderr) - flag = self._fetch_board_flag(board_name) - if flag is None: + primary_flag = self._fetch_board_flag(board_name) + if primary_flag is None: + print(f" no target flag found in {board_name}", file=sys.stderr) continue - arch = _arch_from_flag(flag) - selected_packages = set(flag_to_packages.get(flag, [])) + active = _expand_flags(primary_flag, meta_rules) - # Find emulators available for this board + # Determine which packages are selected for this board + selected_packages: set[str] = set() + for pkg, cond in selects: + if _condition_holds(cond, active): + selected_packages.add(pkg) + + # Map selected packages to emulator names via es_systems.yml emulators: set[str] = set() - for pkg, emus in package_to_emulators.items(): - if pkg in selected_packages: - emulators.update(emus) + for pkg in selected_packages: + for emu in package_to_emulators.get(pkg, []): + emulators.add(emu) + arch = _arch_from_flag(primary_flag) targets[target_key] = { "architecture": arch, "cores": sorted(emulators), } + print( + f" {len(emulators)} emulators ({len(selected_packages)} packages selected)", + file=sys.stderr, + ) return { "platform": "batocera",