chore: drop copies only an install provides

A file that arrives with the emulator is already on the user's machine
and the packs leave it out. 4558 collected copies (1.4 GB) were reached
only by such entries, with no platform, other profile entry or install
manifest needing them. They are removed, and two release-asset copies
leave the ignore list. bundled_only.py now lists every copy of a listed
file, since the database keeps one path per content, and pipeline step
4d fails while one remains.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 10:17:08 +02:00
1 parent f71b00615d
commit ef8b372810
4568 files changed
+291 -6121581

No files matched your search

+29 -1
View File
@@ -19,6 +19,7 @@ import json
from pathlib import Path
from common import (
compute_hashes,
expand_directory_entries,
list_registered_platforms,
load_database,
@@ -70,12 +71,39 @@ def bundled_only(db: dict, profiles: dict, platforms_dir: str, install_dir: Path
else:
needed.add(sha1)
files = db.get("files", {})
return [
rows = [
{"sha1": sha1, "path": files[sha1]["path"], "size": files[sha1].get("size", 0),
"emulators": sorted(emulators)}
for sha1, emulators in sorted(bundled.items(), key=lambda kv: files[kv[0]]["path"])
if sha1 not in needed and sha1 in files
]
return rows + other_copies(rows, Path(db.get("bios_dir") or "bios"))
def other_copies(rows: list[dict], bios_dir: Path) -> list[dict]:
"""The same bytes under other paths of the collection.
The database keeps one path per sha1, and the trees kept whole
(NODEDUP: splitwolf's sprite directories, c64-emu's machine folders)
hold the same file several times. Listing only the indexed path took
five passes to empty splitwolf. A file is hashed only when its size is
one a listed file has.
"""
by_size: dict[int, dict[str, dict]] = {}
for row in rows:
by_size.setdefault(int(row["size"]), {})[row["sha1"]] = row
listed = {row["path"] for row in rows}
found = []
for path in sorted(bios_dir.rglob("*")):
if not path.is_file() or path.is_symlink() or path.as_posix() in listed:
continue
same_size = by_size.get(path.stat().st_size)
if not same_size:
continue
sha1 = compute_hashes(path, frozenset({"sha1"}))["sha1"]
if sha1 in same_size:
found.append({**same_size[sha1], "path": path.as_posix()})
return found
def main() -> int:
-4
View File
@@ -51,10 +51,6 @@ KNOWN_SHARED: dict[str, tuple[frozenset[str], str]] = {
frozenset({"Oric", "Tangerine"}),
"Oric Microdisc ROM, the Oric being a Tangerine machine",
),
"0334b35f164089df7b2a82d46fa0ec6e43fafa90": (
frozenset({"Sinclair", "Timex"}),
"ZX81 overlay, the TS1500 being the same machine for Timex",
),
"286b2e1fb21cc79851da01666db6c0b0e88f25e3": (
frozenset({"Infocom", "ZC"}),
"one soundfont shipped by two interpreters",
+12
View File
@@ -20,6 +20,7 @@ Steps, numbered as the run prints them:
4 generate_pack.py --all ZIP packs
4b generate_pack.py --manifest install manifests
4c --manifest-targets per-target core lists
4d bundled_only.py --strict copies only an emulator install provides
5 consistency check verify counts == pack counts
6 --verify-packs pack integrity, extraction and hashes
7 generate_readme.py README.md and CONTRIBUTING.md
@@ -728,6 +729,17 @@ def main() -> None:
print("\n--- 4c/8 generate target manifests: SKIPPED (--skip-packs) ---")
results["generate_target_manifests"] = SKIPPED
# Step 4d: A collected copy that only an emulator's own install provides.
# Packs leave such files out, so a copy nothing else asks for is weight
# the collection carries for nobody. Read after the manifests, which say
# what an installation ships.
ok, _ = run(
[sys.executable, "scripts/bundled_only.py", "--strict"],
"4d/8 copies only an emulator install provides",
)
results["bundled_only"] = ok
all_ok &= ok
# Step 5: Consistency check. verify describes the full pack, and a run that
# built only a reduced one (--source platform or truth) has nothing to
# compare it with, and says so instead of failing on an empty parse.