mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-30 15:47:20 -05:00
feat: copy BIOS files from a folder with --from
--from FOLDER (repeatable, a ZIP accepted) reads an old SD card, a backup or a Downloads folder: files indexed by size, hashed only when a size matches an entry, ZIP members included. Each file is copied, checked again once written, and the source is never modified. Mac ._ files, system folders and links leading out of the folder are skipped. An install also puts in place the files it finds misplaced in the BIOS folder (bios/bios/, a pack extracted one level too deep, a name in capitals, a ZIP never extracted, one folder up), leaving the user's copy where it is. --check and --dry-run say what a run would copy. Events gain one kind, found, sent before the copy, which a downloaded event then reports. The end of the run says where each file came from.
This commit is contained in:
1 parent
d458cf653a
commit
44cdf2984b
4 files changed
+1059
-64
No files matched your search
+587
-62
@@ -16,6 +16,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import datetime
|
||||
import errno
|
||||
import hashlib
|
||||
@@ -43,7 +44,7 @@ import zipfile
|
||||
import zlib
|
||||
from collections.abc import Callable, Collection, Iterator
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import BinaryIO, NamedTuple, TextIO
|
||||
from typing import BinaryIO, ContextManager, NamedTuple, TextIO
|
||||
|
||||
# Manifests are read from the same ref the bootstrap verified this installer
|
||||
# against, so a file list and the code reading it always come from one commit.
|
||||
@@ -487,7 +488,8 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
|
||||
# stage: manifest (reason: network, not_found, invalid, clock for a
|
||||
# refused certificate under a clock set in the past, or no_ca for no
|
||||
# trusted certificates on the machine) or detect (reason: conflict, two
|
||||
# systems found on one machine).
|
||||
# systems found on one machine) or from (reason: not_found or
|
||||
# not_a_folder, a folder named with --from).
|
||||
"error": ("stage", "reason", "platform", "message"),
|
||||
# state: present (hash verified), wrong_hash or missing.
|
||||
"file": ("platform", "dest", "systems", "size", "state"),
|
||||
@@ -495,6 +497,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
|
||||
"wrong_hash", "bytes_needed"),
|
||||
"check": ("platform", "name", "bios_dir", "files", "present", "missing",
|
||||
"wrong_hash", "bytes_needed", "data_missing", "not_installable"),
|
||||
# A file to install that this machine already holds: misplaced in the
|
||||
# BIOS folder (folder is its bios_dir, reason says why in words) or in
|
||||
# a folder named with --from (reason empty). member names the ZIP member
|
||||
# when path is an archive, else "". Its copy is a downloaded event.
|
||||
"found": ("platform", "dest", "folder", "path", "member", "reason"),
|
||||
"download": ("platform", "files", "bytes"),
|
||||
"progress": ("files_done", "files_total", "bytes_done", "bytes_total"),
|
||||
"downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total",
|
||||
@@ -4172,9 +4179,10 @@ def restore_kept(run: Path, dry_run: bool = False) -> int:
|
||||
|
||||
def _download_one(
|
||||
f: dict, bios_path: Path, verbose: bool = False,
|
||||
progress: Progress | None = None,
|
||||
progress: Progress | None = None, copy: LocalCopy | None = None,
|
||||
) -> tuple[str, bool, str]:
|
||||
"""Download a single file. Returns (dest, success, reason it failed)."""
|
||||
"""Download a single file, or copy it from where copy found it on this
|
||||
machine. Returns (dest, success, reason it failed)."""
|
||||
if _stop.is_set():
|
||||
return str(f.get("dest", "?")), False, "stopped"
|
||||
try:
|
||||
@@ -4188,6 +4196,18 @@ def _download_one(
|
||||
return f["dest"], False, NOT_WRITABLE
|
||||
return f["dest"], False, exc.strerror or str(exc)
|
||||
|
||||
if copy is not None:
|
||||
result = _copy_local(f, bios_path, dest, copy, progress)
|
||||
if result[1]:
|
||||
copy.index.count_copy(copy.misplaced)
|
||||
return result
|
||||
if result[2] in ("stopped", NOT_WRITABLE, READ_ONLY) or result[2].startswith(NOT_KEPT):
|
||||
return result
|
||||
# The copy went or changed since it was read (a card pulled out, a
|
||||
# file another run replaced): the file is fetched as if never found.
|
||||
if verbose:
|
||||
print(f" {f['dest']}: {result[2]}, fetching it instead", file=sys.stderr)
|
||||
|
||||
if LOCAL.pack is not None:
|
||||
return _extract_from_pack(f, bios_path, dest, progress)
|
||||
_network_allowed(f["dest"])
|
||||
@@ -4592,7 +4612,7 @@ def _folder_writable(path: Path) -> bool:
|
||||
# Options that take a value, as a hint gives them back.
|
||||
_VALUED_OPTIONS = (
|
||||
"--system", "--core", "--emulator", "--region", "--target", "--cores-dir", "--save-dir",
|
||||
"--from-pack", "--manifest",
|
||||
"--from-pack", "--from", "--manifest",
|
||||
)
|
||||
|
||||
|
||||
@@ -4781,6 +4801,7 @@ def _report_outcome(
|
||||
omitted_entries: list[dict] | None = None,
|
||||
platform_of: dict[Path, str] | None = None,
|
||||
hints: list[str] | None = None,
|
||||
sources: str = "",
|
||||
) -> None:
|
||||
"""Close the run on what happened, and leave with a status that says so.
|
||||
|
||||
@@ -4853,6 +4874,8 @@ def _report_outcome(
|
||||
if errors:
|
||||
print(f"\nFinished with errors. {_plural(downloaded, 'file')} installed, "
|
||||
f"{_plural(errors, 'file')} failed.")
|
||||
if sources:
|
||||
print(f" {sources}")
|
||||
print(f"Location: {_locations(paths)}")
|
||||
for line in kept + advice:
|
||||
print(line)
|
||||
@@ -4861,6 +4884,8 @@ def _report_outcome(
|
||||
|
||||
print(f"\nDone. {_plural(downloaded, 'file')} installed, "
|
||||
f"{_plural(up_to_date, 'file')} already up to date.")
|
||||
if sources:
|
||||
print(f" {sources}")
|
||||
print(f"Location: {_locations(paths)}")
|
||||
for line in kept:
|
||||
print(line)
|
||||
@@ -4959,6 +4984,7 @@ def _print_plan(
|
||||
to_download: list[dict], bios_path: Path, term: Terminal,
|
||||
trees: list[tuple[dict, dict[str, dict]]] = (),
|
||||
kept: dict[Path, list[int]] | None = None,
|
||||
copied: list[str] = (),
|
||||
) -> None:
|
||||
"""What a real run would fetch, by system, and nothing written.
|
||||
|
||||
@@ -4970,16 +4996,21 @@ def _print_plan(
|
||||
trees = [(entry, needed) for entry, needed in trees if needed]
|
||||
tree_files = sum(len(needed) for _entry, needed in trees)
|
||||
tree_size = sum(meta.get("size", 0) for _entry, needed in trees for meta in needed.values())
|
||||
for line in copied:
|
||||
print(line)
|
||||
if not to_download and not trees:
|
||||
print("\n All files up to date. A real run would download nothing.")
|
||||
return
|
||||
if not copied:
|
||||
print("\n All files up to date. A real run would download nothing.")
|
||||
return
|
||||
print("\n Nothing to download.")
|
||||
parts = []
|
||||
if to_download:
|
||||
parts.append(f"{_plural(len(to_download), 'file')} ({format_size(total)})")
|
||||
if trees:
|
||||
parts.append(f"{_plural(tree_files, 'data file')} ({format_size(tree_size)}, "
|
||||
f"extracted from {_plural(len(trees), 'archive')})")
|
||||
print(f"\nWould download {' and '.join(parts)} to {bios_path}:")
|
||||
if parts:
|
||||
print(f"\nWould download {' and '.join(parts)} to {bios_path}:")
|
||||
by_system: dict[str, list[dict]] = {}
|
||||
for entry in to_download:
|
||||
system = ", ".join(entry_systems(entry)) or "other"
|
||||
@@ -5035,6 +5066,7 @@ def _check_free_space(bios_path: Path, needed: int, plat: str = "", code: int =
|
||||
def download_files(
|
||||
files: list[dict], bios_path: Path, jobs: int = 8, verbose: bool = False,
|
||||
term: Terminal | None = None, progress: Progress | None = None,
|
||||
copies: dict[int, LocalCopy] | None = None,
|
||||
) -> tuple[list[tuple[str, str]], bool]:
|
||||
"""Download files in parallel.
|
||||
|
||||
@@ -5054,7 +5086,8 @@ def download_files(
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
|
||||
future_map = {
|
||||
pool.submit(_download_one, f, bios_path, verbose, progress): f
|
||||
pool.submit(_download_one, f, bios_path, verbose, progress,
|
||||
(copies or {}).get(id(f))): f
|
||||
for f in files
|
||||
}
|
||||
refused = False
|
||||
@@ -6401,11 +6434,14 @@ class PackSource:
|
||||
|
||||
|
||||
class LocalSources:
|
||||
"""Where this run reads instead of the network: a file list, a pack."""
|
||||
"""Where this run reads instead of the network: a file list, a pack, the
|
||||
folders it copies BIOS files from."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.manifest: Path | None = None
|
||||
self.pack: PackSource | None = None
|
||||
# The folders named with --from, read once for every platform.
|
||||
self.folders: list[FolderIndex] = []
|
||||
|
||||
@property
|
||||
def offline(self) -> bool:
|
||||
@@ -6416,6 +6452,7 @@ class LocalSources:
|
||||
self.pack.close()
|
||||
self.manifest = None
|
||||
self.pack = None
|
||||
self.folders = []
|
||||
|
||||
|
||||
LOCAL = LocalSources()
|
||||
@@ -6474,11 +6511,42 @@ def _extract_from_pack(
|
||||
if held is None:
|
||||
return f["dest"], False, NOT_IN_PACK
|
||||
archive, info = held
|
||||
return _write_verified(f, bios_path, dest, lambda: archive.open(info), progress,
|
||||
"the pack")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _open_copy(copy: LocalCopy) -> Iterator[BinaryIO]:
|
||||
"""The bytes of a copy found on this machine, opened for reading only."""
|
||||
if not copy.member:
|
||||
with open(copy.path, "rb") as handle:
|
||||
yield handle
|
||||
return
|
||||
with zipfile.ZipFile(copy.path) as archive, archive.open(copy.member) as handle:
|
||||
yield handle
|
||||
|
||||
|
||||
def _copy_local(
|
||||
f: dict, bios_path: Path, dest: Path, copy: LocalCopy, progress: Progress | None,
|
||||
) -> tuple[str, bool, str]:
|
||||
"""Copy one file found on this machine to its place, checked like a
|
||||
download. Its bytes count as already here, not in the download speed."""
|
||||
return _write_verified(f, bios_path, dest, lambda: _open_copy(copy), progress,
|
||||
f"the copy in {copy.index.label}", local=True)
|
||||
|
||||
|
||||
def _write_verified(
|
||||
f: dict, bios_path: Path, dest: Path,
|
||||
source: Callable[[], ContextManager[BinaryIO]], progress: Progress | None,
|
||||
held_in: str, local: bool = False,
|
||||
) -> tuple[str, bool, str]:
|
||||
"""Write the bytes source opens at dest once they match the list's size
|
||||
and hashes, the file in place kept aside first when it differs."""
|
||||
expected = {name: f[name].lower() for name in ("sha256", "sha1") if f.get(name)}
|
||||
tmp_path: Path | None = None
|
||||
counted = 0
|
||||
try:
|
||||
with archive.open(info) as src, tempfile.NamedTemporaryFile(
|
||||
with source() as src, tempfile.NamedTemporaryFile(
|
||||
mode="wb", dir=dest.parent, prefix=f".{dest.name}.", suffix=".part",
|
||||
delete=False,
|
||||
) as out:
|
||||
@@ -6492,17 +6560,17 @@ def _extract_from_pack(
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > f["size"]:
|
||||
raise ValueError("the pack holds more bytes than the list records")
|
||||
raise ValueError(f"{held_in} holds more bytes than the list records")
|
||||
out.write(chunk)
|
||||
if progress is not None:
|
||||
progress.add(len(chunk))
|
||||
progress.add(len(chunk), resumed=local)
|
||||
counted += len(chunk)
|
||||
if size != f["size"]:
|
||||
raise ValueError(f"the pack holds {size} bytes, the list records {f['size']}")
|
||||
raise ValueError(f"{held_in} holds {size} bytes, the list records {f['size']}")
|
||||
actual = _digest_file(tmp_path, tuple(expected))
|
||||
for name, label in (("sha256", "SHA-256"), ("sha1", "SHA-1")):
|
||||
if name in expected and actual[name] != expected[name]:
|
||||
raise ValueError(f"{label} mismatch in the pack")
|
||||
raise ValueError(f"{label} mismatch in {held_in}")
|
||||
os.chmod(tmp_path, (0o777 if f.get("executable") else 0o666) & ~_UMASK)
|
||||
_clear_read_only(dest)
|
||||
KEPT.keep(*_kept_slot(*_entry_place(bios_path, f)), dest, expected, tmp_path)
|
||||
@@ -6513,9 +6581,9 @@ def _extract_from_pack(
|
||||
return f["dest"], False, "stopped"
|
||||
except _NotKept as exc:
|
||||
return f["dest"], False, f"{NOT_KEPT} ({exc})"
|
||||
except (OSError, ValueError, zipfile.BadZipFile, zlib.error) as exc:
|
||||
except (ValueError, *_MEMBER_ERRORS) as exc:
|
||||
if progress is not None and counted:
|
||||
progress.add(-counted)
|
||||
progress.add(-counted, resumed=local)
|
||||
if isinstance(exc, OSError) and _write_refused(exc):
|
||||
return f["dest"], False, READ_ONLY if _read_only_file(dest) else NOT_WRITABLE
|
||||
return f["dest"], False, str(exc)
|
||||
@@ -6532,23 +6600,85 @@ _ARCHIVE_SUFFIXES = (".zip", ".7z", ".rar")
|
||||
SHOWN_CORES = 4
|
||||
|
||||
|
||||
class FolderIndex:
|
||||
"""The files under a BIOS folder, by name and by size, hashed on demand."""
|
||||
# Folders a system writes on a drive for itself. No BIOS file a person
|
||||
# copied is in them, and a Mac's Spotlight index alone holds thousands.
|
||||
SYSTEM_FOLDERS = frozenset(name.casefold() for name in (
|
||||
".Spotlight-V100", ".Trashes", ".fseventsd", ".TemporaryItems",
|
||||
".DocumentRevisions-V100", "System Volume Information", "$RECYCLE.BIN",
|
||||
))
|
||||
# A Mac copying to a FAT or exFAT card writes ._name beside every file: its
|
||||
# Finder data, 4 KB, never the file.
|
||||
APPLE_DOUBLE = "._"
|
||||
# Errors a member of someone's ZIP can raise on reading: damaged, encrypted
|
||||
# (RuntimeError), or packed with a method zipfile lacks.
|
||||
_MEMBER_ERRORS = (OSError, EOFError, RuntimeError, NotImplementedError,
|
||||
zipfile.BadZipFile, zlib.error)
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
|
||||
class LocalCopy(NamedTuple):
|
||||
"""A file found on this machine with the contents an entry wants."""
|
||||
index: FolderIndex
|
||||
path: Path
|
||||
# The member holding the bytes when path is a ZIP, else "".
|
||||
member: str = ""
|
||||
# In the BIOS folder at a place no entry of the list names: a mistake
|
||||
# the copy fixes. A copy from another entry's place is no mistake.
|
||||
misplaced: bool = False
|
||||
|
||||
def where(self) -> str:
|
||||
"""The copy as the folder it was found in names it."""
|
||||
rel = self.index.relative(self.path)
|
||||
return f"{rel}, inside as {self.member}" if self.member else rel
|
||||
|
||||
|
||||
class FolderIndex:
|
||||
"""The files under a folder, by name and by size, hashed on demand.
|
||||
|
||||
The folder is only read. A link is followed only where it leads to a
|
||||
file inside the folder, so an old card holding a link to the home
|
||||
folder is not walked through it. A ZIP named as the root is read as a
|
||||
folder holding that one archive.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path, label: str = "") -> None:
|
||||
self.label = label or str(root)
|
||||
self.root = root = root.resolve()
|
||||
self.by_name: dict[str, list[Path]] = {}
|
||||
self.by_size: dict[int, list[Path]] = {}
|
||||
self.size: dict[Path, int] = {}
|
||||
self.archives: list[Path] = []
|
||||
# Name a ._ file stands beside (casefolded) -> the ._ files.
|
||||
self.apple_double: dict[str, list[Path]] = {}
|
||||
# The list's destinations, casefolded (an archive among them is not
|
||||
# unpacked for its members) and as the list writes them.
|
||||
self.sealed: set[str] = set()
|
||||
self.listed: set[str] = set()
|
||||
self.truncated = False
|
||||
self.copied = 0
|
||||
self.fixed = 0
|
||||
self._lock = threading.Lock()
|
||||
self._sha1: dict[Path, str] = {}
|
||||
self._digests: dict[tuple[Path, str, str], str] = {}
|
||||
self._members: dict[Path, list[str]] = {}
|
||||
self._member_sizes: dict[int, list[tuple[Path, str]]] | None = None
|
||||
if root.is_file():
|
||||
if root.name.casefold().endswith(".zip"):
|
||||
self.archives.append(root)
|
||||
return
|
||||
seen = 0
|
||||
for folder, dirs, names in os.walk(root):
|
||||
dirs[:] = sorted(d for d in dirs if d != BACKUP_DIR)
|
||||
dirs[:] = sorted(d for d in dirs
|
||||
if d != BACKUP_DIR and d.casefold() not in SYSTEM_FOLDERS)
|
||||
for name in sorted(names):
|
||||
path = Path(folder) / name
|
||||
if name.startswith(APPLE_DOUBLE):
|
||||
self.apple_double.setdefault(name[2:].casefold(), []).append(path)
|
||||
continue
|
||||
if name.startswith(".") and name.endswith(".part"):
|
||||
continue # a download of this installer, not finished
|
||||
try:
|
||||
if path.is_symlink() and not _inside(path, root):
|
||||
continue
|
||||
if not path.is_file():
|
||||
continue
|
||||
size = path.stat().st_size
|
||||
@@ -6561,6 +6691,7 @@ class FolderIndex:
|
||||
self.archives.append(path)
|
||||
seen += 1
|
||||
if seen >= MAX_SCANNED_FILES:
|
||||
self.truncated = True
|
||||
return
|
||||
|
||||
def seed(self, path: Path, sha1: str) -> None:
|
||||
@@ -6575,6 +6706,23 @@ class FolderIndex:
|
||||
self._sha1[path] = ""
|
||||
return self._sha1[path]
|
||||
|
||||
def digest(self, path: Path, algorithm: str, member: str = "") -> str:
|
||||
"""The file's (or the member's) digest, read once, "" when unreadable."""
|
||||
if algorithm == "sha1" and not member:
|
||||
return self.sha1(path)
|
||||
key = (path, member, algorithm)
|
||||
if key not in self._digests:
|
||||
try:
|
||||
if member:
|
||||
with zipfile.ZipFile(path) as archive, archive.open(member) as handle:
|
||||
value = _digest_handle(handle, (algorithm,))[algorithm]
|
||||
else:
|
||||
value = _digest_file(path, (algorithm,))[algorithm]
|
||||
except _MEMBER_ERRORS:
|
||||
value = ""
|
||||
self._digests[key] = value
|
||||
return self._digests[key]
|
||||
|
||||
def holding(self, sizes: set[int], sha1s: set[str]) -> list[Path]:
|
||||
"""Files whose contents are one of sha1s, read only when the size fits."""
|
||||
return [path for size in sorted(sizes) for path in self.by_size.get(size, [])
|
||||
@@ -6590,11 +6738,70 @@ class FolderIndex:
|
||||
self._members[path] = []
|
||||
return self._members[path]
|
||||
|
||||
def _members_of_size(self, size: int) -> list[tuple[Path, str]]:
|
||||
"""(archive, member) of every ZIP member of that size, the
|
||||
archives' directories read once, the first time one is asked."""
|
||||
if self._member_sizes is None:
|
||||
self._member_sizes = {}
|
||||
for path in self.archives:
|
||||
if (not path.name.casefold().endswith(".zip")
|
||||
or self.relative(path).casefold() in self.sealed):
|
||||
continue
|
||||
try:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
infos = archive.infolist()
|
||||
except (OSError, zipfile.BadZipFile):
|
||||
continue
|
||||
for info in infos:
|
||||
if not info.is_dir():
|
||||
self._member_sizes.setdefault(info.file_size, []).append(
|
||||
(path, info.filename))
|
||||
return self._member_sizes.get(size, [])
|
||||
|
||||
def find(self, entry: dict, skip: Collection[Path] = ()) -> LocalCopy | None:
|
||||
"""A copy of the entry's own contents: its size first, then its hash,
|
||||
loose files before ZIP members, a file of the same name first."""
|
||||
size = entry.get("size")
|
||||
if not isinstance(size, int) or isinstance(size, bool):
|
||||
return None
|
||||
algorithm = "sha1" if entry.get("sha1") else "sha256"
|
||||
wanted = str(entry.get(algorithm) or "").lower()
|
||||
if not wanted:
|
||||
return None
|
||||
name = str(entry.get("dest", "")).rsplit("/", 1)[-1].casefold()
|
||||
loose = sorted(self.by_size.get(size, []), key=lambda p: p.name.casefold() != name)
|
||||
for path in loose:
|
||||
if path not in skip and self.digest(path, algorithm) == wanted:
|
||||
return LocalCopy(self, path)
|
||||
members = sorted(self._members_of_size(size),
|
||||
key=lambda item: item[1].rsplit("/", 1)[-1].casefold() != name)
|
||||
for path, member in members:
|
||||
if self.digest(path, algorithm, member) == wanted:
|
||||
return LocalCopy(self, path, member)
|
||||
return None
|
||||
|
||||
def count_copy(self, misplaced: bool = False) -> None:
|
||||
with self._lock:
|
||||
if misplaced:
|
||||
self.fixed += 1
|
||||
else:
|
||||
self.copied += 1
|
||||
|
||||
def relative(self, path: Path) -> str:
|
||||
try:
|
||||
return path.relative_to(self.root).as_posix()
|
||||
rel = path.relative_to(self.root).as_posix()
|
||||
except ValueError:
|
||||
return str(path)
|
||||
return rel if rel != "." else path.name
|
||||
|
||||
|
||||
def _inside(path: Path, root: Path) -> bool:
|
||||
"""Whether a link at path leads to a place under root."""
|
||||
try:
|
||||
Path(os.path.realpath(path)).relative_to(root)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _copies(entry: dict) -> list[dict]:
|
||||
@@ -6656,6 +6863,59 @@ def _content_owner(sha1: str, entry: dict, files: list[dict]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _place_reason(
|
||||
entry: dict, path: Path, index: FolderIndex, expected: Path, known: set[str],
|
||||
contents: str = "",
|
||||
) -> tuple[str, str]:
|
||||
"""(reason, folder too deep) for an entry's file found at path in the
|
||||
BIOS folder, or ("", "") when that place says nothing. contents is
|
||||
", with other contents" for a file of the right name and wrong bytes."""
|
||||
dest = str(entry.get("dest", ""))
|
||||
name = dest.rsplit("/", 1)[-1]
|
||||
rel = index.relative(path)
|
||||
# A dest below the save directory is not below this folder: a copy
|
||||
# here under the same tail is in the wrong place, not too deep.
|
||||
below = entry.get("root") != "save"
|
||||
if below and rel.endswith("/" + dest) and path.name == name:
|
||||
extra = rel[: -len(dest) - 1]
|
||||
top = index.root.name
|
||||
if extra.casefold() == top.casefold():
|
||||
return (f"nested folder: it is at {top}/{rel}{contents}, a "
|
||||
f"{top} folder inside {top}"), extra
|
||||
depth = extra.count("/") + 1
|
||||
levels = "one folder" if depth == 1 else f"{depth} folders"
|
||||
return f"{levels} too deep: it is at {rel}{contents}", extra
|
||||
if path.name.casefold() == name.casefold():
|
||||
if path.parent == expected.parent and path.name != name:
|
||||
return (f"case of the name: the folder holds {path.name}{contents}, "
|
||||
f"the cores open {name}"), ""
|
||||
return f"wrong folder: it is at {rel}{contents}", ""
|
||||
if contents:
|
||||
return "", ""
|
||||
if rel.casefold() in known:
|
||||
return f"its contents are at {rel}: copy that file to {dest}", ""
|
||||
return f"a name no core opens: its contents are at {rel}, rename it {name}", ""
|
||||
|
||||
|
||||
def _copy_reason(entry: dict, copy: LocalCopy, bios_path: Path, known: set[str]) -> str:
|
||||
"""Why a copy found in the BIOS folder is not at the entry's place."""
|
||||
if not copy.misplaced:
|
||||
# np2kai/2608_bd.wav and np2/2608_bd.wav hold the same bytes. Told
|
||||
# "wrong folder", a user moved np2's copy and broke that core.
|
||||
return f"the same file is at {copy.where()}, where the list installs it too"
|
||||
if copy.member:
|
||||
return f"still zipped: it is inside {copy.index.relative(copy.path)}"
|
||||
try:
|
||||
copy.path.relative_to(copy.index.root)
|
||||
except ValueError:
|
||||
return f"one folder too high: it is at {copy.path}"
|
||||
try:
|
||||
expected = _entry_path(bios_path, entry)
|
||||
except ValueError:
|
||||
return ""
|
||||
return _place_reason(entry, copy.path, copy.index, expected, known)[0]
|
||||
|
||||
|
||||
def _found_elsewhere(
|
||||
entry: dict, index: FolderIndex, bios_path: Path, files: list[dict],
|
||||
known: set[str],
|
||||
@@ -6678,32 +6938,15 @@ def _found_elsewhere(
|
||||
same_name = [p for p in index.by_name.get(name.casefold(), []) if p != expected]
|
||||
matching = [p for p in same_name if index.size[p] in sizes and index.sha1(p) in sha1s]
|
||||
for path in matching or same_name:
|
||||
rel = index.relative(path)
|
||||
contents = "" if path in matching else ", with other contents"
|
||||
# A dest below the save directory is not below this folder: a copy
|
||||
# here under the same tail is in the wrong place, not too deep.
|
||||
below = entry.get("root") != "save"
|
||||
if below and rel.endswith("/" + dest) and path.name == name:
|
||||
extra = rel[: -len(dest) - 1]
|
||||
top = index.root.name
|
||||
if extra.casefold() == top.casefold():
|
||||
return (f"nested folder: it is at {top}/{rel}{contents}, a "
|
||||
f"{top} folder inside {top}"), extra
|
||||
depth = extra.count("/") + 1
|
||||
levels = "one folder" if depth == 1 else f"{depth} folders"
|
||||
return f"{levels} too deep: it is at {rel}{contents}", extra
|
||||
if path.parent == expected.parent and path.name != name:
|
||||
return f"case of the name: the folder holds {path.name}{contents}", ""
|
||||
if path.name.casefold() == name.casefold():
|
||||
return f"wrong folder: it is at {rel}{contents}", ""
|
||||
found = _place_reason(entry, path, index, expected, known, contents)
|
||||
if found[0]:
|
||||
return found
|
||||
above = index.root.parent / dest
|
||||
if index.root.parent != index.root and above.is_file():
|
||||
return f"one folder too high: it is at {above}", ""
|
||||
for path in index.holding(sizes, sha1s):
|
||||
rel = index.relative(path)
|
||||
if rel.casefold() in known:
|
||||
return f"its contents are at {rel}: copy that file to {dest}", ""
|
||||
return f"a name no core opens: its contents are at {rel}, rename it {name}", ""
|
||||
return _place_reason(entry, path, index, expected, known)
|
||||
if name.casefold().endswith(_ARCHIVE_SUFFIXES):
|
||||
return "", ""
|
||||
stem = name.rsplit(".", 1)[0].casefold()
|
||||
@@ -6716,6 +6959,10 @@ def _found_elsewhere(
|
||||
return f"still zipped: it is inside {index.relative(path)}", ""
|
||||
if lowered.rsplit(".", 1)[0] == stem:
|
||||
return f"still zipped: {index.relative(path)} needs extracting", ""
|
||||
for path in index.apple_double.get(name.casefold(), []):
|
||||
if path.parent == expected.parent:
|
||||
return (f"only a Mac's ._ file: {path.name} holds Finder data, not the "
|
||||
f"file itself. Copy {name} again"), ""
|
||||
return "", ""
|
||||
|
||||
|
||||
@@ -6747,7 +6994,7 @@ class Diagnosis(NamedTuple):
|
||||
|
||||
def diagnose(
|
||||
missing: list[dict], mismatched: list[dict], files: list[dict], bios_path: Path,
|
||||
verified: list[dict] | None = None,
|
||||
verified: list[dict] | None = None, index: FolderIndex | None = None,
|
||||
) -> Diagnosis:
|
||||
"""The reason --check gives for each file it flags, from what the folder
|
||||
holds and what the list records: id(entry) -> reason, and the folders
|
||||
@@ -6757,13 +7004,8 @@ def diagnose(
|
||||
deep: dict[str, int] = {}
|
||||
if not bios_path.is_dir() or not (missing or mismatched):
|
||||
return Diagnosis(reasons, deep)
|
||||
index = FolderIndex(bios_path)
|
||||
for entry in verified or []:
|
||||
if entry.get("sha1"):
|
||||
try:
|
||||
index.seed(_entry_path(bios_path, entry), entry["sha1"].lower())
|
||||
except ValueError:
|
||||
continue
|
||||
if index is None:
|
||||
index = bios_folder_index(bios_path, files, verified or [])
|
||||
known = _known_destinations(files)
|
||||
for entry in missing:
|
||||
reason, extra = _found_elsewhere(entry, index, bios_path, files, known)
|
||||
@@ -6785,21 +7027,26 @@ def diagnose(
|
||||
def check_lines(
|
||||
missing: list[dict], mismatched: list[dict], files: list[dict], manifest: dict,
|
||||
bios_path: Path, every: bool, verified: list[dict] | None = None,
|
||||
index: FolderIndex | None = None, copies: dict[int, LocalCopy] | None = None,
|
||||
) -> list[str]:
|
||||
"""The files --check flags, each with why and where its cores read it.
|
||||
|
||||
"missing: scph5501.bin" left a user with the file one folder too deep,
|
||||
or still in its ZIP, to guess. A flagged file with a reason is always
|
||||
listed. Plain missing ones stop at ten unless every is set.
|
||||
listed, and so is one a folder named with --from holds. Plain missing
|
||||
ones stop at ten unless every is set.
|
||||
"""
|
||||
found = diagnose(missing, mismatched, files, bios_path, verified)
|
||||
found = diagnose(missing, mismatched, files, bios_path, verified, index)
|
||||
copies = copies or {}
|
||||
known = _known_destinations(files)
|
||||
lines = []
|
||||
for extra, count in sorted(found.deep.items(), key=lambda item: -item[1]):
|
||||
if count >= 3:
|
||||
lines.append(f" {_plural(count, 'file')} sit too deep, in "
|
||||
f"{bios_path / extra}: move what it holds up into {bios_path}.")
|
||||
explained = [e for e in missing if id(e) in found.reasons]
|
||||
plain = [e for e in missing if id(e) not in found.reasons]
|
||||
lines += _check_copy_lines(missing + mismatched, copies, index)
|
||||
explained = [e for e in missing if id(e) in found.reasons or id(e) in copies]
|
||||
plain = [e for e in missing if id(e) not in found.reasons and id(e) not in copies]
|
||||
shown = plain if every else plain[:10]
|
||||
|
||||
def block(label: str, entry: dict) -> list[str]:
|
||||
@@ -6807,6 +7054,16 @@ def check_lines(
|
||||
rows = [f" {label}: {entry['dest']}{game}"]
|
||||
if id(entry) in found.reasons:
|
||||
rows.append(f" {found.reasons[id(entry)]}")
|
||||
copy = copies.get(id(entry))
|
||||
if copy is not None and copy.index is index and not copy.misplaced:
|
||||
rows.append(f" a run without --check copies it from {copy.where()}")
|
||||
elif copy is not None and copy.index is index:
|
||||
where = _copy_reason(entry, copy, bios_path, known)
|
||||
if where and where != found.reasons.get(id(entry)):
|
||||
rows.append(f" the right file: {where}")
|
||||
rows.append(" a run without --check copies it into place")
|
||||
elif copy is not None:
|
||||
rows.append(f" in {copy.index.label}: {copy.where()}")
|
||||
rows.append(f" {opened_line(entry, manifest, bios_path)}")
|
||||
return rows
|
||||
|
||||
@@ -6822,6 +7079,199 @@ def check_lines(
|
||||
return lines
|
||||
|
||||
|
||||
def bios_folder_index(
|
||||
bios_path: Path, files: list[dict], verified: list[dict],
|
||||
) -> FolderIndex | None:
|
||||
"""The BIOS folder read for files misplaced in it. The files the check
|
||||
verified are not read again: their SHA-1 is the one the list records.
|
||||
An archive the list installs is not unpacked for its members: a MAME
|
||||
set holding a ROM a core also reads loose is that core's to open."""
|
||||
if not bios_path.is_dir():
|
||||
return None
|
||||
index = FolderIndex(bios_path)
|
||||
for entry in verified:
|
||||
if entry.get("sha1"):
|
||||
try:
|
||||
index.seed(_entry_path(bios_path, entry), entry["sha1"].lower())
|
||||
except ValueError:
|
||||
continue
|
||||
index.sealed = _known_destinations(files)
|
||||
index.listed = {str(f.get("dest", "")) for f in files}
|
||||
return index
|
||||
|
||||
|
||||
def _above(entry: dict, index: FolderIndex) -> LocalCopy | None:
|
||||
"""The entry's file at its place one folder above the BIOS folder."""
|
||||
if entry.get("root") == "save" or index.root.parent == index.root:
|
||||
return None
|
||||
try:
|
||||
above = index.root.parent / _safe_relative_path(entry.get("dest"), "dest")
|
||||
if above.stat().st_size != entry.get("size") or not above.is_file():
|
||||
return None
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
algorithm = "sha1" if entry.get("sha1") else "sha256"
|
||||
if index.digest(above, algorithm) != str(entry.get(algorithm, "")).lower():
|
||||
return None
|
||||
return LocalCopy(index, above)
|
||||
|
||||
|
||||
def plan_local_copies(
|
||||
entries: list[dict], bios_path: Path, bios_index: FolderIndex | None,
|
||||
folders: list[FolderIndex],
|
||||
) -> dict[int, LocalCopy]:
|
||||
"""id(entry) -> where this machine already holds the entry's bytes: the
|
||||
BIOS folder itself first, a file misplaced in it, then each folder named
|
||||
with --from in turn. Every copy is found by size, then by hash."""
|
||||
if not entries or (bios_index is None and not folders):
|
||||
return {}
|
||||
# A file this run replaces is no source: its place is being written.
|
||||
written = set()
|
||||
for entry in entries:
|
||||
try:
|
||||
written.add(Path(os.path.realpath(_entry_path(bios_path, entry))))
|
||||
except ValueError:
|
||||
continue
|
||||
copies: dict[int, LocalCopy] = {}
|
||||
for entry in entries:
|
||||
copy = None
|
||||
if bios_index is not None:
|
||||
copy = bios_index.find(entry, written) or _above(entry, bios_index)
|
||||
if copy is not None:
|
||||
copy = copy._replace(misplaced=_misplaced(entry, copy, bios_index))
|
||||
for index in folders:
|
||||
if copy is not None:
|
||||
break
|
||||
copy = index.find(entry, written)
|
||||
if copy is not None:
|
||||
copies[id(entry)] = copy
|
||||
return copies
|
||||
|
||||
|
||||
def _misplaced(entry: dict, copy: LocalCopy, index: FolderIndex) -> bool:
|
||||
"""Whether a copy in the BIOS folder sits where no other entry of the
|
||||
list goes: GBA_BIOS.BIN for gba_bios.bin is a mistake, np2/2608_bd.wav
|
||||
for np2kai/2608_bd.wav is the other core's file."""
|
||||
if copy.member:
|
||||
return True
|
||||
rel = index.relative(copy.path)
|
||||
if rel in index.listed:
|
||||
return False
|
||||
own = str(entry.get("dest", "")).casefold()
|
||||
return rel.casefold() not in index.sealed or rel.casefold() == own
|
||||
|
||||
|
||||
def _check_copy_lines(
|
||||
entries: list[dict], copies: dict[int, LocalCopy], bios_index: FolderIndex | None,
|
||||
) -> list[str]:
|
||||
"""What --check says a run would copy instead of downloading."""
|
||||
fixed = sum(1 for e in entries if id(e) in copies and copies[id(e)].misplaced)
|
||||
lines = []
|
||||
if fixed:
|
||||
verb = "is" if fixed == 1 else "are"
|
||||
them = "it" if fixed == 1 else "them"
|
||||
lines.append(f" {fixed} of these files {verb} in the BIOS folder at the wrong place: "
|
||||
f"a run without --check copies {them} where the cores open {them}, and "
|
||||
f"leaves your {'copy' if fixed == 1 else 'copies'} as "
|
||||
f"{'it is' if fixed == 1 else 'they are'}.")
|
||||
for label, count in _counts_by_folder(entries, copies, bios_index):
|
||||
them = "it" if count == 1 else "them"
|
||||
lines.append(f" {label} holds {count} of these files: a run without --check "
|
||||
f"copies {them}.")
|
||||
return lines
|
||||
|
||||
|
||||
def _counts_by_folder(
|
||||
entries: list[dict], copies: dict[int, LocalCopy], bios_index: FolderIndex | None,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""(folder, files) for each --from folder that gives some of entries."""
|
||||
counts: dict[str, int] = {}
|
||||
for entry in entries:
|
||||
copy = copies.get(id(entry))
|
||||
if copy is not None and copy.index is not bios_index:
|
||||
counts[copy.index.label] = counts.get(copy.index.label, 0) + 1
|
||||
return list(counts.items())
|
||||
|
||||
|
||||
def copy_plan_lines(
|
||||
entries: list[dict], copies: dict[int, LocalCopy], bios_index: FolderIndex | None,
|
||||
bios_path: Path, files: list[dict], every: bool, planned: bool,
|
||||
) -> list[str]:
|
||||
"""The files a run copies from this machine, by folder: the misplaced
|
||||
ones with the reason each was not where its cores open it."""
|
||||
known = _known_destinations(files)
|
||||
# By --from folder, "" for the misplaced files and "=" for the ones the
|
||||
# BIOS folder holds at another entry's place.
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for entry in entries:
|
||||
copy = copies.get(id(entry))
|
||||
if copy is not None:
|
||||
key = (copy.index.label if copy.index is not bios_index
|
||||
else "" if copy.misplaced else "=")
|
||||
groups.setdefault(key, []).append(entry)
|
||||
lines = []
|
||||
for key in sorted(groups, key=lambda k: (k != "", k != "=")):
|
||||
chosen = groups[key]
|
||||
sized = f"{_plural(len(chosen), 'file')} ({format_size(_bytes(chosen))})"
|
||||
if key == "":
|
||||
head = (f"Would fix {sized} misplaced in the BIOS folder, copied where the "
|
||||
"cores open them:" if planned else
|
||||
f"Fixing {sized} misplaced in the BIOS folder: copied where the cores "
|
||||
"open them, your copies left as they are.")
|
||||
elif key == "=":
|
||||
head = (f"Would copy {sized} the BIOS folder already holds at another "
|
||||
"place of the list:" if planned else
|
||||
f"Copying {sized} the BIOS folder already holds at another place "
|
||||
"of the list.")
|
||||
else:
|
||||
head = (f"Would copy {sized} from {key}:" if planned else
|
||||
f"Copying {sized} from {key}, each one checked by hash.")
|
||||
lines.append(f"\n {head}" if planned else f" {head}")
|
||||
for entry in chosen if every else chosen[:10]:
|
||||
copy = copies[id(entry)]
|
||||
why = (_copy_reason(entry, copy, bios_path, known) if key == ""
|
||||
else f"from {copy.where()}" if key == "=" else copy.where())
|
||||
lines.append(f" {entry['dest']}: {why}" if why else f" {entry['dest']}")
|
||||
if len(chosen) > 10 and not every:
|
||||
lines.append(f" ... and {len(chosen) - 10} more (--verbose lists them all)")
|
||||
return lines
|
||||
|
||||
|
||||
def emit_copies(
|
||||
plat: str, entries: list[dict], copies: dict[int, LocalCopy],
|
||||
bios_index: FolderIndex | None, bios_path: Path, files: list[dict],
|
||||
) -> None:
|
||||
"""One found event per file this machine holds, before it is copied."""
|
||||
known = _known_destinations(files)
|
||||
for entry in entries:
|
||||
copy = copies.get(id(entry))
|
||||
if copy is None:
|
||||
continue
|
||||
reason = (_copy_reason(entry, copy, bios_path, known)
|
||||
if copy.index is bios_index else "")
|
||||
EVENTS.emit("found", platform=plat, dest=entry["dest"], folder=copy.index.label,
|
||||
path=str(copy.path), member=copy.member, reason=reason)
|
||||
|
||||
|
||||
def sources_line(
|
||||
folders: list[tuple[str, int]], fixed: int, downloaded: int, missing: int,
|
||||
fetched: str = "downloaded", shared: int = 0,
|
||||
) -> str:
|
||||
"""Where the files of a run that copied some came from, in words:
|
||||
"38 files copied from E:\\, 4 downloaded, 2 still missing." """
|
||||
parts = [(count, f"copied from {label}") for label, count in folders]
|
||||
parts.append((fixed, "fixed from copies already in the BIOS folder"))
|
||||
parts.append((shared, "copied from another place of the BIOS folder"))
|
||||
if not any(count for count, _what in parts):
|
||||
return ""
|
||||
parts += [(downloaded, fetched), (missing, "still missing")]
|
||||
words = []
|
||||
for count, what in parts:
|
||||
if count:
|
||||
words.append(f"{_plural(count, 'file') if not words else count} {what}")
|
||||
return ", ".join(words) + "."
|
||||
|
||||
|
||||
# Below this a download is small enough that narrowing it is not worth a line.
|
||||
BULK_HINT_FROM = 100 * 1024 * 1024
|
||||
|
||||
@@ -7082,6 +7532,15 @@ def main() -> None:
|
||||
"(the others are found beside it) or the folder holding them. "
|
||||
"Makes no network request, and checks every file's hash",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from",
|
||||
action="append",
|
||||
type=Path,
|
||||
metavar="FOLDER",
|
||||
help="copy the BIOS files found in FOLDER (an old SD card, a backup, a "
|
||||
"ZIP) instead of downloading them: every file recognised by its "
|
||||
"hash, never moved. Repeat for several folders",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--manifest",
|
||||
type=Path,
|
||||
@@ -7126,7 +7585,7 @@ def main() -> None:
|
||||
"cores_dir", "save_dir", "system", "core", "region", "no_game_data",
|
||||
"list_systems",
|
||||
"list_cores", "romm_firmware", "list_emulators", "events", "quiet",
|
||||
"yes", "verbose", "from_pack", "manifest")
|
||||
"yes", "verbose", "from_pack", "manifest", "from")
|
||||
given = [name for name in alone if getattr(args, name)]
|
||||
if args.jobs != 8:
|
||||
given.append("jobs")
|
||||
@@ -7139,6 +7598,12 @@ def main() -> None:
|
||||
if args.from_pack and args.target:
|
||||
parser.error("--target reads a core list from the network, which --from-pack "
|
||||
"does not use")
|
||||
if getattr(args, "from") and (
|
||||
args.list_platforms or args.list_targets or args.list_systems
|
||||
or args.list_cores or args.list_emulators
|
||||
):
|
||||
parser.error("--from copies files into the BIOS folder, which a listing "
|
||||
"does not write")
|
||||
if (args.from_pack or args.manifest) and (args.list_platforms or args.list_targets):
|
||||
parser.error("--from-pack and --manifest give a file list, which --list-platforms "
|
||||
"and --list-targets do not read")
|
||||
@@ -7193,6 +7658,7 @@ def _open_local_sources(args: argparse.Namespace) -> None:
|
||||
"""Take the file list and the pack this run was given, and the platform
|
||||
they are for when none is named."""
|
||||
LOCAL.reset()
|
||||
_open_from_folders(getattr(args, "from", None) or [])
|
||||
if args.manifest is not None:
|
||||
LOCAL.manifest = args.manifest
|
||||
named = local_manifest_platform(args.manifest)
|
||||
@@ -7227,6 +7693,28 @@ def _open_local_sources(args: argparse.Namespace) -> None:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _open_from_folders(folders: list[Path]) -> None:
|
||||
"""Read each folder named with --from, or stop on one that is not there."""
|
||||
for folder in folders:
|
||||
label = str(folder)
|
||||
folder = Path(os.path.expanduser(label))
|
||||
zipped = folder.is_file() and folder.name.casefold().endswith(".zip")
|
||||
if not (folder.is_dir() or zipped):
|
||||
reason = "not_a_folder" if folder.exists() else "not_found"
|
||||
what = ("does not exist" if reason == "not_found"
|
||||
else "is neither a folder nor a ZIP")
|
||||
message = f"{label} {what}: --from names the folder to copy BIOS files from"
|
||||
EVENTS.emit("error", stage="from", reason=reason, platform="", message=message)
|
||||
print(f"Error: {message}.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
index = FolderIndex(folder, label)
|
||||
LOCAL.folders.append(index)
|
||||
print(f" Copying BIOS files from: {label}")
|
||||
if index.truncated:
|
||||
print(f" Read the first {MAX_SCANNED_FILES:,} files of {label} only. Name "
|
||||
"the folder that holds the BIOS files to search it all.")
|
||||
|
||||
|
||||
def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest") -> None:
|
||||
"""Say that a --dest folder does not exist before anything goes there.
|
||||
|
||||
@@ -7521,6 +8009,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
failures: list[tuple[str, str]] = []
|
||||
installed_paths: list[Path] = []
|
||||
bulk_lines: list[str] = []
|
||||
# The BIOS folders read for misplaced files, one per platform.
|
||||
bios_indexes: list[FolderIndex] = []
|
||||
|
||||
# --target is checked for every platform before anything is written, as
|
||||
# --installed-cores is below: EmuDeck installed 3.2 GB and only then was
|
||||
@@ -7820,6 +8310,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
|
||||
# Mismatched files need re-download
|
||||
to_download.extend(mismatched)
|
||||
# The files this machine already holds: misplaced in the BIOS folder
|
||||
# (bios/bios/, a pack extracted into a folder of its own, a name in
|
||||
# capitals, still zipped), or in a folder named with --from.
|
||||
bios_index = (bios_folder_index(bios_path, files, up_to_date)
|
||||
if to_download else None)
|
||||
if bios_index is not None:
|
||||
bios_indexes.append(bios_index)
|
||||
copies = plan_local_copies(to_download, bios_path, bios_index, LOCAL.folders)
|
||||
hint = [] if LOCAL.pack is not None else bulk_hint(
|
||||
to_download, manifest,
|
||||
systems_chosen=bool(selection and selection[0]),
|
||||
@@ -7843,6 +8341,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
sum(len(state[1]) for _entry, state in data_states), len(omitted_files),
|
||||
chosen=selection[0] if selection else None,
|
||||
)
|
||||
emit_copies(plat_name, to_download, copies, bios_index, bios_path, files)
|
||||
total_verified += len(up_to_date)
|
||||
|
||||
if args.check:
|
||||
@@ -7859,7 +8358,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
print(line)
|
||||
print(f"\n {_plural(len(to_download), 'file')} to download:")
|
||||
for line in check_lines(missing, mismatched, files, manifest, bios_path,
|
||||
args.verbose, up_to_date):
|
||||
args.verbose, up_to_date, bios_index, copies):
|
||||
print(line)
|
||||
print(" Run the same command without --check to fetch them; "
|
||||
"a file with a wrong hash is replaced, and the one in place "
|
||||
@@ -7891,19 +8390,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
{"dest": f"{entry['dest']}/{rel}" if entry.get("dest") else rel}
|
||||
for entry, needed in trees for rel in needed
|
||||
], bios_path)
|
||||
_print_plan(to_download, bios_path, term, trees, kept)
|
||||
_print_plan([f for f in to_download if id(f) not in copies], bios_path, term,
|
||||
trees, kept, copy_plan_lines(to_download, copies, bios_index,
|
||||
bios_path, files, args.verbose, True))
|
||||
for line in hint:
|
||||
print(line)
|
||||
continue
|
||||
|
||||
if to_download:
|
||||
dl_size = sum(f.get("size", 0) for f in to_download)
|
||||
fetched = [f for f in to_download if id(f) not in copies]
|
||||
fetch_size = _bytes(fetched)
|
||||
if (
|
||||
interactive
|
||||
and not args.yes
|
||||
and dl_size >= CONFIRM_FROM_BYTES
|
||||
and fetch_size >= CONFIRM_FROM_BYTES
|
||||
and not _confirm(
|
||||
f"\nDownload {format_size(dl_size)} to {bios_path}? [Y/n] "
|
||||
f"\nDownload {format_size(fetch_size)} to {bios_path}? [Y/n] "
|
||||
)
|
||||
):
|
||||
# A beginner who just passed the c entry was sent to a
|
||||
@@ -7915,7 +8418,17 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
files_pending += len(to_download)
|
||||
continue
|
||||
EVENTS.emit("download", platform=plat_name, files=len(to_download), bytes=dl_size)
|
||||
if LOCAL.pack is not None:
|
||||
if copies:
|
||||
print()
|
||||
for line in copy_plan_lines(to_download, copies, bios_index, bios_path,
|
||||
files, args.verbose, False):
|
||||
print(line)
|
||||
rest = (f" and {'copying' if LOCAL.pack is not None else 'downloading'} "
|
||||
f"{len(fetched)}" if fetched else "")
|
||||
where = f" from {LOCAL.pack.name}" if fetched and LOCAL.pack is not None else ""
|
||||
print(f"\nCopying {_plural(len(copies), 'file')} found on this machine"
|
||||
f"{rest}{where} ({format_size(dl_size)})...")
|
||||
elif LOCAL.pack is not None:
|
||||
print(f"\nCopying {len(to_download)} files ({format_size(dl_size)}) "
|
||||
f"from {LOCAL.pack.name}...")
|
||||
else:
|
||||
@@ -7943,7 +8456,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
progress = Progress(len(to_download), dl_size, term)
|
||||
failed, interrupted = download_files(
|
||||
to_download, bios_path, jobs=args.jobs, verbose=args.verbose,
|
||||
term=term, progress=progress,
|
||||
term=term, progress=progress, copies=copies,
|
||||
)
|
||||
bulk_lines += hint
|
||||
total_downloaded += progress.installed
|
||||
@@ -8048,9 +8561,21 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
omitted_entries=omitted_seen,
|
||||
platform_of={path: plat for plat, path in platforms},
|
||||
hints=bulk_lines,
|
||||
sources=_run_sources(bios_indexes, total_downloaded, total_errors),
|
||||
)
|
||||
|
||||
|
||||
def _run_sources(bios_indexes: list[FolderIndex], installed: int, failed: int) -> str:
|
||||
"""The end line saying which folder gave how many of the files."""
|
||||
folders = [(index.label, index.copied) for index in LOCAL.folders]
|
||||
fixed = sum(index.fixed for index in bios_indexes)
|
||||
shared = sum(index.copied for index in bios_indexes)
|
||||
fetched = installed - fixed - shared - sum(count for _label, count in folders)
|
||||
return sources_line(folders, fixed, max(fetched, 0), failed,
|
||||
"copied from the pack" if LOCAL.pack is not None else "downloaded",
|
||||
shared)
|
||||
|
||||
|
||||
def _line_buffered_stdout() -> None:
|
||||
"""Flush stdout line by line, as stderr is.
|
||||
|
||||
|
||||
Reference in new issue
Block a user