diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 80cba7fb..33895de7 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -39,9 +39,8 @@ jobs: - name: Validate BIOS files id: validate run: | - files=$(cat /tmp/changed_files.txt) - if [ -n "$files" ]; then - python scripts/validate_pr.py --markdown $files > /tmp/report.md 2>&1 || true + if [ -s /tmp/changed_files.txt ]; then + xargs python scripts/validate_pr.py --markdown < /tmp/changed_files.txt > /tmp/report.md 2>&1 || true else echo "No BIOS files changed" > /tmp/report.md fi diff --git a/scripts/auto_fetch.py b/scripts/auto_fetch.py index 02f741bc..83fdd56a 100644 --- a/scripts/auto_fetch.py +++ b/scripts/auto_fetch.py @@ -142,7 +142,7 @@ def step2_scan_branches(entry: dict) -> bytes | None: ) for filepath in result.stdout.strip().split("\n"): - if filepath.endswith(f"/{name}") or filepath == name or filepath.endswith(name): + if os.path.basename(filepath) == name: try: blob = subprocess.run( ["git", "show", f"{ref}:{filepath}"], diff --git a/scripts/common.py b/scripts/common.py index 45dd635f..aa2bcf23 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -93,7 +93,11 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") - for system in config.get("systems", {}).values(): for group_name in system.get("includes", []): if group_name in shared_groups: - system.setdefault("files", []).extend(shared_groups[group_name]) + existing_names = {f.get("name") for f in system.get("files", [])} + for gf in shared_groups[group_name]: + if gf.get("name") not in existing_names: + system.setdefault("files", []).append(gf) + existing_names.add(gf.get("name")) return config diff --git a/scripts/dedup.py b/scripts/dedup.py index 4bfe345c..a5f91435 100644 --- a/scripts/dedup.py +++ b/scripts/dedup.py @@ -55,7 +55,8 @@ def path_priority(path: str) -> tuple: def _in_nodedup_dir(path: str) -> bool: """Check if a file is inside a no-dedup directory.""" - return any(nodedup in path for nodedup in NODEDUP_DIRS) + parts = Path(path).parts + return any(nodedup in parts for nodedup in NODEDUP_DIRS) def scan_duplicates(bios_dir: str) -> dict[str, list[str]]: diff --git a/scripts/download.py b/scripts/download.py index fdf296f8..b09d5607 100644 --- a/scripts/download.py +++ b/scripts/download.py @@ -207,7 +207,7 @@ Examples: print(f" - {p}") else: print("No platform packs found in latest release") - except Exception as e: + except (urllib.error.URLError, urllib.error.HTTPError, OSError, json.JSONDecodeError) as e: print(f"Error: {e}") return @@ -216,7 +216,7 @@ Examples: try: release = get_latest_release() - except Exception as e: + except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e: print(f"Error fetching release info: {e}") sys.exit(1) diff --git a/scripts/generate_db.py b/scripts/generate_db.py index cae3b354..a094e539 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -263,7 +263,7 @@ def _collect_all_aliases(files: dict) -> dict: matched = md5_to_sha1[r.md5] if matched: _add_alias(basename, matched) - except (ImportError, ConnectionError) as e: + except (ImportError, ConnectionError, OSError): pass # Identical content named differently across platforms/cores diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index b7588baa..66578481 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -51,10 +51,13 @@ def fetch_large_file(name: str, dest_dir: str = ".cache/large") -> str | None: try: req = urllib.request.Request(url, headers={"User-Agent": "retrobios-pack/1.0"}) with urllib.request.urlopen(req, timeout=300) as resp: - data = resp.read() - os.makedirs(dest_dir, exist_ok=True) - with open(cached, "wb") as f: - f.write(data) + os.makedirs(dest_dir, exist_ok=True) + with open(cached, "wb") as f: + while True: + chunk = resp.read(65536) + if not chunk: + break + f.write(chunk) return cached except (urllib.error.URLError, urllib.error.HTTPError): return None @@ -403,7 +406,9 @@ def _group_identical_platforms(platforms: list[str], platforms_dir: str) -> list for fe in system.get("files", []): dest = fe.get("destination", fe.get("name", "")) full_dest = f"{base_dest}/{dest}" if base_dest else dest - entries.append(full_dest) + sha1 = fe.get("sha1", "") + md5 = fe.get("md5", "") + entries.append(f"{full_dest}|{sha1}|{md5}") fingerprint = hashlib.sha1("|".join(sorted(entries)).encode()).hexdigest() fingerprints.setdefault(fingerprint, []).append(platform) diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index cb6a3926..550cdfd8 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -34,7 +34,7 @@ def load_platform_configs(platforms_dir: str) -> dict: config = load_platform_config(f.stem, platforms_dir) if config: configs[f.stem] = config - except Exception as e: + except (yaml.YAMLError, OSError) as e: print(f"Warning: {f.name}: {e}", file=sys.stderr) return configs diff --git a/scripts/list_platforms.py b/scripts/list_platforms.py index 08faa9d3..eb35dff3 100644 --- a/scripts/list_platforms.py +++ b/scripts/list_platforms.py @@ -13,6 +13,7 @@ Usage: from __future__ import annotations +import argparse import json import os import sys @@ -58,9 +59,11 @@ def list_platforms(include_archived: bool = False) -> list[str]: def main(): - include_all = "--all" in sys.argv + parser = argparse.ArgumentParser(description="List available platforms") + parser.add_argument("--all", action="store_true", help="Include archived platforms") + args = parser.parse_args() - platforms = list_platforms(include_archived=include_all) + platforms = list_platforms(include_archived=args.all) if not platforms: print("No platform configs found", file=sys.stderr) diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index b24c8af9..2d56f000 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -133,7 +133,9 @@ class Scraper(BaseScraper): string_char = None clean = [] for j, ch in enumerate(line): - if ch in ('"', "'") and not in_string: + if ch in ('"', "'") and j > 0 and line[j - 1] == '\\': + clean.append(ch) + elif ch in ('"', "'") and not in_string: in_string = True string_char = ch clean.append(ch) diff --git a/scripts/verify.py b/scripts/verify.py index 0bec149e..02a029b1 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -155,13 +155,21 @@ def verify_entry_md5(file_entry: dict, local_path: str | None) -> dict: return {"name": name, "status": Status.MISSING, "expected_md5": expected_md5} if zipped_file: + found_in_zip = False for md5_candidate in md5_list or [""]: result = check_inside_zip(local_path, zipped_file, md5_candidate) if result == Status.OK: return {"name": name, "status": Status.OK, "path": local_path} + if result != "not_in_zip": + found_in_zip = True + reason = ( + f"{zipped_file} not found inside ZIP" + if not found_in_zip + else f"{zipped_file} MD5 mismatch inside ZIP" + ) return { "name": name, "status": Status.UNTESTED, "path": local_path, - "reason": f"{zipped_file} MD5 mismatch inside ZIP", + "reason": reason, } if not md5_list: