fix: batocera targets scraper es_systems path and condition parser

- Fix es_systems.yml URL (batocera-emulationstation -> batocera-es-system)
- Replace if/endif block parser with select-if condition parser that
  matches the actual Config.in structure (select PACKAGE if CONDITION)
- Add Kconfig boolean condition checker (handles &&, ||, !, parentheses)
- Add meta-flag expansion (X86_64_ANY, GLES3, ROCKCHIP_GLES3, etc.)
  iterated to fixpoint for chained derivations
- Fix es_systems.yml parser for the actual dict format with requireAnyOf

scripts/scraper/targets/batocera_targets_scraper.py

@@ -1,9 +1,10 @@
 """Scraper for Batocera per-board emulator availability.
 
 Sources (batocera-linux/batocera.linux):
-  - configs/batocera-*.board  — board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_*
+  - configs/batocera-*.board  -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_*
-  - package/batocera/core/batocera-system/Config.in — flag-to-package mapping
+  - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines
-  - es_systems.yml — emulator-to-requireAnyOf flag mapping
+  - package/batocera/emulationstation/batocera-es-system/es_systems.yml
+    -- emulator requireAnyOf flag mapping
 """
 from __future__ import annotations
 
@@ -24,12 +25,9 @@ PLATFORM_NAME = "batocera"
 GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents"
 RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master"
 
-CONFIG_IN_URL = (
+CONFIG_IN_URL = f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in"
-    f"{RAW_BASE}/package/batocera/core/batocera-system/Config.in"
-)
 ES_SYSTEMS_URL = (
-    f"{RAW_BASE}/package/batocera/emulationstation/batocera-emulationstation/"
+    f"{RAW_BASE}/package/batocera/emulationstation/batocera-es-system/es_systems.yml"
-    "es_systems.yml"
 )
 
 _HEADERS = {
@@ -38,19 +36,24 @@ _HEADERS = {
 }
 
 _TARGET_FLAG_RE = re.compile(r'^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y', re.MULTILINE)
-_REQUIRE_ANYOF_RE = re.compile(
+
-    r'requireAnyOf\s*:\s*\[([^\]]+)\]', re.MULTILINE
+# Matches: select BR2_PACKAGE_FOO  (optional: if CONDITION)
+# Condition may span multiple lines (backslash continuation)
+_SELECT_RE = re.compile(
+    r'^\s+select\s+(BR2_PACKAGE_\w+)'   # package being selected
+    r'(?:\s+if\s+((?:[^\n]|\\\n)+?))?'  # optional "if CONDITION" (may continue with \)
+    r'(?:\s*#[^\n]*)?$',                # optional trailing comment
+    re.MULTILINE,
 )
 
-
+# Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif"
-def _arch_from_flag(flag: str) -> str:
+_META_BLOCK_RE = re.compile(
-    """Guess architecture from board flag name."""
+    r'^if\s+((?:[^\n]|\\\n)+?)\n'       # condition (may span lines via \)
-    low = flag.lower()
+    r'(?:.*?\n)*?'                       # optional lines before the config
-    if "x86_64" in low or "x86-64" in low:
+    r'\s+config\s+(BR2_PACKAGE_\w+)'    # derived flag name
-        return "x86_64"
+    r'.*?^endif',                        # end of block
-    if "x86" in low and "64" not in low:
+    re.MULTILINE | re.DOTALL,
-        return "x86"
+)
-    return "aarch64"
 
 
 def _fetch(url: str, headers: dict | None = None) -> str | None:
@@ -75,53 +78,175 @@ def _fetch_json(url: str) -> list | dict | None:
         return None
 
 
-def _parse_config_in(text: str) -> dict[str, list[str]]:
+def _normalise_condition(raw: str) -> str:
-    """Parse Config.in: map BR2_PACKAGE_BATOCERA_TARGET_* flags to packages."""
+    """Strip backslash-continuations and collapse whitespace."""
-    flag_to_packages: dict[str, list[str]] = {}
+    return re.sub(r'\\\n\s*', ' ', raw).strip()
-    # Find blocks: if BR2_PACKAGE_BATOCERA_TARGET_X ... select BR2_PACKAGE_Y
+
-    block_re = re.compile(
+
-        r'if\s+(BR2_PACKAGE_BATOCERA_TARGET_\w+)(.*?)endif',
+def _tokenise(condition: str) -> list[str]:
-        re.DOTALL,
+    """Split a Kconfig condition into tokens: flags, !, &&, ||, (, )."""
-    )
+    token_re = re.compile(r'&&|\|\||!|\(|\)|BR2_\w+|"[^"]*"')
-    select_re = re.compile(r'select\s+(BR2_PACKAGE_\w+)')
+    return token_re.findall(condition)
-    for m in block_re.finditer(text):
+
-        flag = m.group(1)
+
-        block = m.group(2)
+def _check_condition(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
-        packages = select_re.findall(block)
+    """Recursive descent check of a Kconfig boolean expression."""
-        flag_to_packages.setdefault(flag, []).extend(packages)
+    return _check_or(tokens, pos, active)
-    return flag_to_packages
+
+
+def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
+    left, pos = _check_and(tokens, pos, active)
+    while pos < len(tokens) and tokens[pos] == '||':
+        pos += 1
+        right, pos = _check_and(tokens, pos, active)
+        left = left or right
+    return left, pos
+
+
+def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
+    left, pos = _check_not(tokens, pos, active)
+    while pos < len(tokens) and tokens[pos] == '&&':
+        pos += 1
+        right, pos = _check_not(tokens, pos, active)
+        left = left and right
+    return left, pos
+
+
+def _check_not(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
+    if pos < len(tokens) and tokens[pos] == '!':
+        pos += 1
+        val, pos = _check_atom(tokens, pos, active)
+        return not val, pos
+    return _check_atom(tokens, pos, active)
+
+
+def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
+    if pos >= len(tokens):
+        return True, pos
+    tok = tokens[pos]
+    if tok == '(':
+        pos += 1
+        val, pos = _check_or(tokens, pos, active)
+        if pos < len(tokens) and tokens[pos] == ')':
+            pos += 1
+        return val, pos
+    if tok.startswith('BR2_'):
+        pos += 1
+        return tok in active, pos
+    if tok.startswith('"'):
+        pos += 1
+        return True, pos
+    # Unknown token — treat as true to avoid false negatives
+    pos += 1
+    return True, pos
+
+
+def _condition_holds(condition: str, active: frozenset[str]) -> bool:
+    """Return True if a Kconfig boolean condition holds for the given active flags."""
+    if not condition:
+        return True
+    norm = _normalise_condition(condition)
+    tokens = _tokenise(norm)
+    if not tokens:
+        return True
+    try:
+        result, _ = _check_condition(tokens, 0, active)
+        return result
+    except Exception:
+        return True  # conservative: include on parse failure
+
+
+def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
+    """Return [(derived_flag, condition_str)] from top-level if/endif blocks.
+
+    These define derived flags like BR2_PACKAGE_BATOCERA_TARGET_X86_64_ANY,
+    BR2_PACKAGE_BATOCERA_GLES3, etc.
+    """
+    results: list[tuple[str, str]] = []
+    for m in _META_BLOCK_RE.finditer(text):
+        cond = _normalise_condition(m.group(1))
+        flag = m.group(2)
+        results.append((flag, cond))
+    return results
+
+
+def _expand_flags(primary_flag: str, meta_rules: list[tuple[str, str]]) -> frozenset[str]:
+    """Given a board's primary flag, expand to all active derived flags.
+
+    Iterates until stable (handles chained derivations like X86_64_ANY -> X86_ANY).
+    """
+    active: set[str] = {primary_flag}
+    changed = True
+    while changed:
+        changed = False
+        for derived, cond in meta_rules:
+            if derived not in active and _condition_holds(cond, frozenset(active)):
+                active.add(derived)
+                changed = True
+    return frozenset(active)
+
+
+def _parse_selects(text: str) -> list[tuple[str, str]]:
+    """Parse all 'select PACKAGE [if CONDITION]' lines from Config.in.
+
+    Returns [(package, condition)] where condition is '' if unconditional.
+    """
+    results: list[tuple[str, str]] = []
+    for m in _SELECT_RE.finditer(text):
+        pkg = m.group(1)
+        cond = _normalise_condition(m.group(2) or '')
+        results.append((pkg, cond))
+    return results
 
 
 def _parse_es_systems(text: str) -> dict[str, list[str]]:
-    """Parse es_systems.yml: map emulator name to list of requireAnyOf flags."""
+    """Parse es_systems.yml: map BR2_PACKAGE_* flag -> list of emulator names.
+
+    The file is a dict keyed by system name. Each system has:
+      emulators:
+        <emulator_group>:
+          <core_name>: {requireAnyOf: [BR2_PACKAGE_FOO]}
+    """
     try:
         data = yaml.safe_load(text)
     except yaml.YAMLError:
         return {}
 
-    emulator_flags: dict[str, list[str]] = {}
     if not isinstance(data, dict):
-        return emulator_flags
+        return {}
 
-    systems = data.get("systems", data) if "systems" in data else data
+    package_to_emulators: dict[str, list[str]] = {}
-    if not isinstance(systems, list):
-        # Could be a dict
-        systems = list(systems.values()) if isinstance(systems, dict) else []
 
-    for system in systems:
+    for _system_name, system_data in data.items():
-        if not isinstance(system, dict):
+        if not isinstance(system_data, dict):
             continue
-        for emulator_entry in system.get("emulators", []):
+        emulators = system_data.get("emulators")
-            if not isinstance(emulator_entry, dict):
+        if not isinstance(emulators, dict):
+            continue
+        for _group_name, group_data in emulators.items():
+            if not isinstance(group_data, dict):
                 continue
-            for emu_name, emu_data in emulator_entry.items():
+            for core_name, core_data in group_data.items():
-                if not isinstance(emu_data, dict):
+                if not isinstance(core_data, dict):
                     continue
-                require = emu_data.get("requireAnyOf", [])
+                require = core_data.get("requireAnyOf", [])
-                if isinstance(require, list):
+                if not isinstance(require, list):
-                    emulator_flags.setdefault(emu_name, []).extend(require)
+                    continue
+                for pkg_flag in require:
+                    if isinstance(pkg_flag, str):
+                        package_to_emulators.setdefault(pkg_flag, []).append(core_name)
 
-    return emulator_flags
+    return package_to_emulators
+
+
+def _arch_from_flag(flag: str) -> str:
+    """Guess architecture from board flag name."""
+    low = flag.lower()
+    if "x86_64" in low or "zen3" in low:
+        return "x86_64"
+    if "x86" in low:
+        return "x86"
+    return "aarch64"
 
 
 class Scraper(BaseTargetScraper):
@@ -159,44 +284,55 @@ class Scraper(BaseTargetScraper):
             print("  warning: no boards found", file=sys.stderr)
 
         print("  fetching Config.in...", file=sys.stderr)
-        config_in_text = _fetch(CONFIG_IN_URL)
+        config_in_text = _fetch(CONFIG_IN_URL) or ""
-        flag_to_packages: dict[str, list[str]] = {}
+
-        if config_in_text:
+        meta_rules = _parse_meta_flags(config_in_text)
-            flag_to_packages = _parse_config_in(config_in_text)
+        selects = _parse_selects(config_in_text)
+        print(
+            f"  parsed {len(meta_rules)} meta-flag rules, {len(selects)} select lines",
+            file=sys.stderr,
+        )
 
         print("  fetching es_systems.yml...", file=sys.stderr)
-        es_text = _fetch(ES_SYSTEMS_URL)
+        es_text = _fetch(ES_SYSTEMS_URL) or ""
-        emulator_flags: dict[str, list[str]] = {}
+        package_to_emulators = _parse_es_systems(es_text)
-        if es_text:
+        print(
-            emulator_flags = _parse_es_systems(es_text)
+            f"  parsed {len(package_to_emulators)} package->emulator mappings",
-
+            file=sys.stderr,
-        # Build reverse index: package -> emulators
+        )
-        package_to_emulators: dict[str, list[str]] = {}
-        for emu, flags in emulator_flags.items():
-            for flag in flags:
-                package_to_emulators.setdefault(flag, []).append(emu)
 
         targets: dict[str, dict] = {}
         for board_name in sorted(boards):
             target_key = board_name.removeprefix("batocera-").removesuffix(".board")
             print(f"  processing {target_key}...", file=sys.stderr)
-            flag = self._fetch_board_flag(board_name)
+            primary_flag = self._fetch_board_flag(board_name)
-            if flag is None:
+            if primary_flag is None:
+                print(f"    no target flag found in {board_name}", file=sys.stderr)
                 continue
 
-            arch = _arch_from_flag(flag)
+            active = _expand_flags(primary_flag, meta_rules)
-            selected_packages = set(flag_to_packages.get(flag, []))
 
-            # Find emulators available for this board
+            # Determine which packages are selected for this board
+            selected_packages: set[str] = set()
+            for pkg, cond in selects:
+                if _condition_holds(cond, active):
+                    selected_packages.add(pkg)
+
+            # Map selected packages to emulator names via es_systems.yml
             emulators: set[str] = set()
-            for pkg, emus in package_to_emulators.items():
+            for pkg in selected_packages:
-                if pkg in selected_packages:
+                for emu in package_to_emulators.get(pkg, []):
-                    emulators.update(emus)
+                    emulators.add(emu)
 
+            arch = _arch_from_flag(primary_flag)
             targets[target_key] = {
                 "architecture": arch,
                 "cores": sorted(emulators),
             }
+            print(
+                f"    {len(emulators)} emulators ({len(selected_packages)} packages selected)",
+                file=sys.stderr,
+            )
 
         return {
             "platform": "batocera",