chore: drop copies only an install provides

A file that arrives with the emulator is already on the user's machine
and the packs leave it out. 4558 collected copies (1.4 GB) were reached
only by such entries, with no platform, other profile entry or install
manifest needing them. They are removed, and two release-asset copies
leave the ignore list. bundled_only.py now lists every copy of a listed
file, since the database keeps one path per content, and pipeline step
4d fails while one remains.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 10:17:08 +02:00
1 parent f71b00615d
commit ef8b372810
4568 files changed
+291 -6121581

No files matched your search

+29 -1
View File
@@ -19,6 +19,7 @@ import json
from pathlib import Path
from common import (
compute_hashes,
expand_directory_entries,
list_registered_platforms,
load_database,
@@ -70,12 +71,39 @@ def bundled_only(db: dict, profiles: dict, platforms_dir: str, install_dir: Path
else:
needed.add(sha1)
files = db.get("files", {})
return [
rows = [
{"sha1": sha1, "path": files[sha1]["path"], "size": files[sha1].get("size", 0),
"emulators": sorted(emulators)}
for sha1, emulators in sorted(bundled.items(), key=lambda kv: files[kv[0]]["path"])
if sha1 not in needed and sha1 in files
]
return rows + other_copies(rows, Path(db.get("bios_dir") or "bios"))
def other_copies(rows: list[dict], bios_dir: Path) -> list[dict]:
"""The same bytes under other paths of the collection.
The database keeps one path per sha1, and the trees kept whole
(NODEDUP: splitwolf's sprite directories, c64-emu's machine folders)
hold the same file several times. Listing only the indexed path took
five passes to empty splitwolf. A file is hashed only when its size is
one a listed file has.
"""
by_size: dict[int, dict[str, dict]] = {}
for row in rows:
by_size.setdefault(int(row["size"]), {})[row["sha1"]] = row
listed = {row["path"] for row in rows}
found = []
for path in sorted(bios_dir.rglob("*")):
if not path.is_file() or path.is_symlink() or path.as_posix() in listed:
continue
same_size = by_size.get(path.stat().st_size)
if not same_size:
continue
sha1 = compute_hashes(path, frozenset({"sha1"}))["sha1"]
if sha1 in same_size:
found.append({**same_size[sha1], "path": path.as_posix()})
return found
def main() -> int: