diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 98f52c7dc..5353269cd 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -150,6 +150,13 @@ jobs: npm run build --prefix website python scripts/validate_site.py --site-dir website/dist + # The one-liner the site serves: each copy pins its own installer and + # says what a refusal means for it. Validation runs these too, but in + # another workflow this one never waits for: a copy that refuses every + # install must stop the deploy, the last good pair staying served. + - name: Check the served bootstraps + run: python -m unittest tests.test_site_bootstrap + # The queries a visitor types must reach the page that answers them, # read through the site's own search box, and the home page must stay # under its weight budget. A miss blocks the deploy. The browser driver diff --git a/install.ps1 b/install.ps1 index d847512e3..5313743a8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,12 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "3f450bbd97735e20ce25bd4a7d89e94994d6162edaec108a63f8861452143d2c" + # What a refused installer most likely means for this copy. The copy the + # site serves reads a commit's installer, which no cache serves stale, + # and Invoke-WebRequest throws on a cut transfer before the hash is read: + # the site's build rewrites this line there (scripts/sitebootstrap.py). + $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." + $defaultInstallSha256 = "48a5fca25303757fc2874262c1d176992bd7340da5c59f58c6e911383bf54817" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } @@ -58,8 +63,7 @@ throw ($mismatch + "{0} is set in this session and decides which installer is accepted: " + "remove it to use the installer this script names, or set it to match.") -f ($overrides -join " and ") } - throw ($mismatch + "The download was most likely cut short: try again. Within minutes of an update " + - "of this project the two files can come from caches of different ages. Waiting five minutes settles that.") + throw ($mismatch + $mismatchAdvice) } $installer = $temporary } diff --git a/install.py b/install.py index d58e2e279..b525f9ac4 100644 --- a/install.py +++ b/install.py @@ -6084,7 +6084,7 @@ def _report_outcome( say(f"Location that was being written to: {_locations(paths)}") for line in kept + advice: say(line) - say("Run the same command again once that is resolved. " + say(_retry_words(failures) or "Run the same command again once that is resolved. " "Files already installed are kept.") sys.exit(2) @@ -6098,7 +6098,10 @@ def _report_outcome( say(f"Location: {_locations(paths)}") for line in kept + advice: say(line) - if not all(str(reason).startswith(NOT_KEPT) for _dest, reason in failures or []): + retry = _retry_words(failures) + if retry: + say(retry) + elif not all(str(reason).startswith(NOT_KEPT) for _dest, reason in failures or []): say("Run the same command again to retry only those.") sys.exit(1) @@ -6145,9 +6148,33 @@ def _offline_remedy(folders: list[str] | None = None, named: bool = False) -> st f"again with a pack (--from-pack) or {offline_undo()}.") -def _check_fetch_line(fetch: int, beside: str) -> str: +def _retry_words(failures: list | None) -> str: + """What to do when every failure is a file the pack does not hold: a + second run fails them the same way, and "run it again to retry only + those" sent people round (code_audit_21 P3-3). "" otherwise.""" + lacking = [dest for dest, reason in failures or [] if str(reason) == NOT_IN_PACK] + if not lacking or len(lacking) != len([ + dest for dest, reason in failures or [] if not str(reason).startswith(NOT_KEPT)]): + return "" + name = LOCAL.pack.name if LOCAL.pack is not None else "The pack" + return (f"{name} does not hold {'it' if len(lacking) == 1 else 'them'}: a newer pack, " + "or a run without --from-pack, installs " + f"{'it' if len(lacking) == 1 else 'them'}.") + + +def _check_fetch_line(fetch: int, beside: str, lacking: int = 0) -> str: """What --check says a run without it does with the files not in place: - offline, those it cannot fetch are said so, not "run it to fetch them".""" + offline, those it cannot fetch are said so, not "run it to fetch them". + With a pack, those it holds are copied and those it lacks are said lacking + (lacking): "fetch them" was said with a pack and no network.""" + if LOCAL.pack is not None: + held = fetch - lacking + if not held: + return (f" {LOCAL.pack.name} holds none of them: a newer pack, or a run " + "without --from-pack, installs them.") + return (f" Run the same command without --check to copy {held} of them from " + f"{LOCAL.pack.name}. A file with a wrong hash is replaced, and the one in " + f"place kept in {BACKUP_DIR} beside {beside}.") if _offline_only(): if not fetch: return " Run the same command without --check to copy them into place." @@ -7212,7 +7239,51 @@ def drive_targets(plat: str) -> list[tuple[str, Path]]: targets.append((f"{_drive_folder(mount, below)} ({what})", found[1])) else: targets.append((f"{_drive_folder(mount, 'bios')} (card or drive)", mount / "bios")) - return targets + # A handheld's card no file names first: its bios folder is where its + # system reads (unnamed_cards). + cards = {mount / "bios" for mount in unnamed_cards()} + return sorted(targets, key=lambda target: target[1] not in cards) + + +# The folders a handheld card's games part holds, one per console: the +# names people type for a system (SYSTEM_SHORT_NAMES) and the ArkOS and +# AmberELEC roms folders those miss. +_CONSOLE_FOLDERS = frozenset({*SYSTEM_SHORT_NAMES, "arcade", "atari2600", "atarilynx", + "fbneo", "gamegear", "mame", "mastersystem", "megadrive", + "neogeo", "pcengine", "psp", "segacd", "wonderswan"}) + + +def unnamed_cards() -> list[Path]: + """The mounted drives no file names whose root holds a bios folder and + console folders: ArkOS's EASYROMS, AmberELEC's GAMES. Which system runs + them cannot be told (their own files are on a part a computer does not + open), but they are a handheld's card: "Nothing was found here" over + one, and two presses of Enter filled this computer's folder for a + RetroArch that was not there (tui_audit_14 P2-3).""" + cards = [] + for mount in _mounted_cards(): + if (card_system(mount) is not None or share_doubt(mount)[0] + or not _is_dir(mount / "bios")): + continue + if any(name.casefold() in _CONSOLE_FOLDERS and _is_dir(mount / name) + for name in _listed(mount)): + cards.append(mount) + return cards + + +def unnamed_card_words(cards: list[Path], tail: str = ".") -> str: + """What the first screen and line mode say of those cards.""" + names = ", ".join(_mount_name(mount) for mount in cards) + what = "A card is" if len(cards) == 1 else "Cards are" + return f"{what} in the reader ({names}): choose its system{tail}" + + +def say_unnamed_cards() -> None: + """Line mode's words for the cards no file names, where it says what + was not found here.""" + cards = unnamed_cards() + if cards: + print(f" {unnamed_card_words(cards)}") def _drive_folder(mount: Path, below: str) -> str: @@ -7267,9 +7338,17 @@ def ask_platform_folder(os_type: str, plat: str, ask: bool = True, name = display_name(plat) shared = plat in _SHARE_FOLDERS and found is None default = found if found is not None else None if shared else _default_dest(os_type, plat) + # A handheld's card in the reader and no setup found here: Enter puts + # the files on the card, never in this computer's folder for a RetroArch + # that is not there (tui_audit_14 P2-3). + cards = [] if shared or found is not None else unnamed_cards() print(f"\nWhich folder do {name}'s BIOS files go to?") if shared: print(f" {share_words(plat)}") + elif cards: + default = cards[0] / "bios" + print(f" The card in the reader keeps them in its bios folder: " + f"{shown_path(default)}.") elif found is not None: print(f" {name} reads its BIOS files from: {shown_path(found)}.") elif default is not None: @@ -7280,13 +7359,21 @@ def ask_platform_folder(os_type: str, plat: str, ask: bool = True, # rows: never its root, and never for a system that reads another list. targets = (share_targets(plat) if shared else [*(other_side(os_type, plat) if found is None else []), *drive_targets(plat)]) + if cards: + # The card first, this computer's usual folder after it. + targets = [*targets, (f"{side_name(os_type, plat)}'s usual folder", + _default_dest(os_type, plat))] + targets = [target for target in targets if target[1] is not None] for index, (label, folder) in enumerate(targets, 1): - made = "" if _is_dir(folder) else ", created" + # Said of a folder that is missing, as the screen says it: "created" + # was said of folders nobody had created (code_audit_21 P3-7). + missing = "" if _is_dir(folder) else "does not exist yet" # A label that spells its folder (the Windows side's) is not # followed by it again. shown = shown_path(folder) - print(f" {index}) {label}" + (f" ({shown}{made})" if shown not in label - else f" ({made[2:]})" if made else "")) + print(f" {index}) {label}" + ( + f" ({shown}{', ' + missing if missing else ''})" if shown not in label + else f" ({missing})" if missing else "")) left_out = "" if shared else drives_left_out(plat) if left_out: print(f" {left_out}") @@ -7295,7 +7382,8 @@ def ask_platform_folder(os_type: str, plat: str, ask: bool = True, why = ("" if default is not None or shared else " (no default, a folder above could not be opened)" if _default_set_aside(os_type, plat) else " (no default on this system)") - enter = (f"the folder {name} reads" if found is not None else "the default above") + enter = (f"the folder {name} reads" if found is not None + else "the card's bios folder" if cards else "the default above") prompt = (f"BIOS folder{numbers}, q to quit (Enter for {enter}): " if default else f"BIOS folder{numbers}, q to quit{why}: ") usage = f" Type a folder{', a number' if targets else ''}, or q to quit." @@ -8714,6 +8802,13 @@ def pack_lacks(entries: list[dict]) -> list[dict]: return [] if LOCAL.pack is None else [e for e in entries if LOCAL.pack.find(e) is None] +def pack_holds_trees(data_dirs: list[dict]) -> bool: + """Whether the pack given to this run holds a file of any of these data + trees, which an install takes from it.""" + return LOCAL.pack is not None and any( + next(LOCAL.pack.tree(entry.get("dest", "")), None) is not None for entry in data_dirs) + + def pack_lacks_words(count: int, total: int) -> str: """The pack's missing files, said before the plan names a size.""" name = LOCAL.pack.name if LOCAL.pack is not None else "the pack" @@ -12134,6 +12229,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # RetroArch first, as the screen offers them. Its folder question # alone left ArkOS, muOS and JELOS out of reach (tui_audit_11 P1-2). print(f" {display_name(args.platform)} was not found here.") + say_unnamed_cards() platforms = _prompt_manual_platform( os_type, not (args.check or args.dry_run or args.yes), True, f"Which setup is the {display_name(args.platform)} list for?", setups, mounted) @@ -12142,6 +12238,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # filled with a card mounted in the reader, and with nothing # found here, without a question. print(f" {display_name(args.platform)} was not found here.") + say_unnamed_cards() platforms = [(args.platform, ask_platform_folder( os_type, args.platform, not (args.check or args.dry_run or args.yes)))] elif getattr(args, "platform_from_list", False): @@ -12235,7 +12332,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: if not platforms: # The screen's words where a person answers: "platform" was said # nowhere else. - print(" Nothing was found here." if interactive and not listing + unnamed = unnamed_cards() if interactive and not listing else [] + print(f" {unnamed_card_words(unnamed)}" + if unnamed else " Nothing was found here." if interactive and not listing else " No supported platform detected.") EVENTS.emit("detect", os=os_type, os_name=_os_name(), platforms=[]) # Standalone emulators read their BIOS from folders of their own, @@ -12769,14 +12868,21 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for line in split_pack_lines(bios_path): print(line) offline = _offline_only() + # What the pack lacks is said before the count, as the + # screen says it (code_audit_21 P3-3). + lacking = pack_lacks([f for f in to_download if id(f) not in copies]) + if lacking: + print(f"\n {pack_lacks_words(len(lacking), len(to_download) - len(copies))}") print(f"\n {_plural(len(to_download), 'file')} " - + ("not in place:" if offline else "to download:")) + + ("not in place:" if offline or LOCAL.pack is not None + else "to download:")) for line in check_lines(missing, mismatched, files, manifest, bios_path, args.verbose, up_to_date, bios_index, copies): print(line) for line in unreadable_lines(unread, args.verbose): print(line) - print(_check_fetch_line(len(to_download) - len(copies), "the BIOS folder")) + print(_check_fetch_line(len(to_download) - len(copies), "the BIOS folder", + len(lacking))) check_pending += len(to_download) else: print("\n All files up to date.") @@ -12804,7 +12910,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: {"dest": f"{entry['dest']}/{rel}" if entry.get("dest") else rel} for entry, needed in trees for rel in needed ], bios_path) - _print_plan([f for f in to_download if id(f) not in copies], bios_path, term, + planned = [f for f in to_download if id(f) not in copies] + # Said before the plan promises a count and a size, as an + # install says it: the dry run promised 23 files where the pack + # held 20 (code_audit_21 P3-3). + lacking = {id(f) for f in pack_lacks(planned)} + if lacking: + print(f"\n {pack_lacks_words(len(lacking), len(planned))}") + _print_plan([f for f in planned if id(f) not in lacking], bios_path, term, trees, kept, copy_plan_lines(to_download, copies, bios_index, bios_path, files, args.verbose, True), verbose=args.verbose, manifest=manifest) @@ -12834,6 +12947,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: lacking = {id(f) for f in pack_lacks(fetched)} if lacking: print(f"\n {pack_lacks_words(len(lacking), len(fetched))}") + if lacking and len(lacking) == len(to_download) and not pack_holds_trees(data_dirs): + # Nothing here can come from the pack: no "Copying 0 files", no + # folder made, each file failed as not in it. + failures.extend((f["dest"], NOT_IN_PACK) for f in to_download) + total_errors += len(to_download) + files_pending += len(to_download) + total_up_to_date += len(up_to_date) + continue dl_size = sum(f.get("size", 0) for f in to_download if id(f) not in lacking) fetch_size = _bytes([f for f in fetched if id(f) not in lacking]) if not to_download: @@ -12868,8 +12989,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for line in copy_plan_lines(to_download, copies, bios_index, bios_path, files, args.verbose, False): print(line) + held = len(fetched) - len(lacking) rest = (f" and {'copying' if LOCAL.pack is not None else 'downloading'} " - f"{len(fetched)}" if fetched else "") + f"{held}" if held else "") where = f" from {LOCAL.pack.name}" if fetched and LOCAL.pack is not None else "" # Named by the folder they come from, as the line above # names it: "found on this machine" followed "from E:\". @@ -12884,8 +13006,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print(f"\nCopying {_plural(len(copies), 'file')}{'' if rest else sized} " f"{origin}{rest}{where}{sized if rest else ''}...") elif LOCAL.pack is not None: - print(f"\nCopying {len(to_download)} files ({format_size(dl_size)}) " - f"from {LOCAL.pack.name}...") + # The files the pack holds: "Copying 23 files (85.0 KB)" sat + # beside the size of 20 (code_audit_21 P3-3). + print(f"\nCopying {_plural(len(to_download) - len(lacking), 'file')} " + f"({format_size(dl_size)}) from {LOCAL.pack.name}...") else: print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...") print(" Press Ctrl+C to stop. Running the same command again " @@ -12978,7 +13102,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: line = f" {entry['ref']}: {counts['installed']} installed, " \ f"{counts['present']} already up to date" if counts["changed"] or counts["absent"]: - line += (f", {counts['changed'] + counts['absent']} no longer in " + # From a pack, the pack is the source that lacks them. + line += (f", {counts['changed'] + counts['absent']} not in " + f"{LOCAL.pack.name}" if LOCAL.pack is not None else + f", {counts['changed'] + counts['absent']} no longer in " "the source (the pack has them)") print(line) if counts["error"]: @@ -13671,6 +13798,9 @@ class TuiState: # (platform, folder) of the setups that are a card in a reader, not # this machine: a setup chosen for that system asks for its card. self.mounted_setups: set = set() + # The mounted drives that are a handheld's card no file names + # (unnamed_cards): their bios folder comes first at Which folder?. + self.unnamed: list = [] self.notes: list = [] # The folders detection could not look in, one line each. self.refused: list = [] @@ -14515,14 +14645,22 @@ def tui_rows(state: TuiState) -> list[TuiRow]: # The Windows side's folder said missing where it is: taken, the # plan said RetroArch reads a folder no RetroArch had (tui_audit_13 # P3-9). + # Every row whose folder is missing says so, in both modes: the + # Windows row alone did (code_audit_21 P3-7). drives = [] if state.folder_kind == "share" else [ - TuiRow("drive", label, value=folder) - for label, folder in [*((label + ("" if _is_dir(folder) - else " (does not exist yet)"), folder) - for label, folder in ( - other_side(state.os_type, state.platform) - if state.folder_kind == "usual" else [])), + TuiRow("drive", label + ("" if _is_dir(folder) else " (does not exist yet)"), + value=folder) + for label, folder in [*(other_side(state.os_type, state.platform) + if state.folder_kind == "usual" else []), *drive_targets(state.platform)]] + cards = {mount / "bios" for mount in state.unnamed} + if state.folder_kind == "usual" and any(row.value in cards for row in drives): + # A handheld's card in the reader, no setup found here: the + # card first, under A (tui_audit_14 P2-3). + first = [row for row in drives if row.value in cards] + return first + rows + [TuiRow("browse", "Browse for a folder"), + TuiRow("type", "Type a path")] + [ + row for row in drives if row not in first] return rows + [TuiRow("browse", "Browse for a folder"), TuiRow("type", "Type a path")] + drives # Only the cards whose own files say they run the system: A on the @@ -14671,8 +14809,9 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: files = {event["dest"] for event in state.files} if (LOCAL.pack is not None and files and not _plan_copies(state, check) and len(_tui_pack_lacks(state, sorted(files))) == len(files) - and not check.get("data_missing")): - # The pack holds none of them: Install would fail every one. + and not (check.get("data_missing") and _tui_pack_trees(state))): + # The pack holds none of them, nor the data files: Install would + # fail every one (tui_audit_14 P3-1). rows = [] if _on_its_card(state) and (state.facts.get("writable") is False or _short_of_space(state, check)): @@ -14734,8 +14873,15 @@ def _tui_tally(state: TuiState, events: Iterable[dict]) -> Tally: def _tui_estimating(state: TuiState) -> bool: """Whether the plan counts the systems ticked since the check, not the check itself.""" - return bool(_choice_line(state) and state.systems and state.check is not None - and not _offline_only()) + if not (_choice_line(state) and state.systems and state.check is not None): + return False + if _offline_only(): + # Offline the estimate is of the ticks alone: a folder to copy from + # is counted by the check it starts. + def ticked(flags: list) -> list: + return [flag for flag in flags if flag[0] == "--system"] + return ticked(tui_flags(state)) != ticked(state.checked_flags) + return True def _choice_entries(state: TuiState, hidden: bool = False) -> list[dict]: @@ -17418,7 +17564,10 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: headline = (f"{_plural(data, 'data file')} to fetch{data_size}" if data else "Everything is in place") elif lacking and not fetch and not copied: - headline = pack_lacks_words(len(lacking), len(lacking)) + # The data files too, when the pack holds none of them: "8252 + # files" said nothing of the 2636 it cannot give either. + lacked = len(lacking) + (data if data and not _tui_pack_trees(state) else 0) + headline = pack_lacks_words(lacked, lacked) wrong = check["wrong_hash"] # Where some were found, the plan names the rest by kind ("Not in old: # 10 BIOS files and 6 game data files.", _view_plan), and the remedy @@ -17531,6 +17680,12 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: return lines[:at] + offline_lines + lines[at:] +def _tui_pack_trees(state: TuiState) -> bool: + """Whether the pack given to the run holds a file of the list's data + trees.""" + return pack_holds_trees((state.manifest or {}).get("data_directories") or []) + + def _tui_pack_lacks(state: TuiState, dests: list[str]) -> list[str]: """The destinations of a plan the pack given to the run does not hold.""" if LOCAL.pack is None or not dests: @@ -17648,7 +17803,11 @@ def _view_platforms(state: TuiState, width: int, height: int) -> tuple: # A handheld's owner knows its device, not its system's name: the # filter finds it, and a pad that cannot type finds the cards right # after RetroArch (_cards_first). - pinned.append("Nothing was found here. Choose the setup, or type your device's " + pinned.append(unnamed_card_words(state.unnamed, ", or type your device's name (/)." + if state.pad in ("both", "keys") else + ", handhelds after RetroArch.") + if state.unnamed else + "Nothing was found here. Choose the setup, or type your device's " "name (/)." if state.pad in ("both", "keys") else "Nothing was found here. Choose the setup: handhelds come after " "RetroArch.") @@ -17703,6 +17862,10 @@ def _view_folder(state: TuiState, width: int, height: int) -> tuple: f"{side_name(state.os_type, state.platform)} usually reads its BIOS " "files from:") before = [lead, str(state.folder_default)] + if (state.folder_kind == "usual" and tui_rows(state) + and tui_rows(state)[0].value in {mount / "bios" for mount in state.unnamed}): + before = ["The card in the reader keeps them in its bios folder, first below. " + f"{lead}", str(state.folder_default)] elif state.folder_kind == "closed": title = "Which folder?" before = [f"Choose another folder for {name}: browse for it or type it."] @@ -17853,6 +18016,26 @@ def _view_plan(state: TuiState, width: int, height: int) -> tuple: lines[0] = _styled(f"About {_plural(len(chosen) + data, 'file')}, " f"{format_size(sum(e['size'] for e in chosen) + (data_size or 0))}" f", for {len(state.ticked)} of {len(state.systems)} systems", "title") + if _offline_only(): + # Offline, what the folders copied from hold of the systems + # ticked: the plan counted the whole list beside "For your 1 of + # 11 systems" (code_audit_21 P3-5). + took = {event["dest"] for events in _tui_found_elsewhere(state).values() + for event in events} + held = [event for event in chosen if event["dest"] in took] + rest = len(chosen) - len(held) + data + lines[0] = _styled( + f"About {_plural(len(held), 'file')}, " + f"{format_size(sum(e['size'] for e in held))}, for {len(state.ticked)} of " + f"{len(state.systems)} systems" + + (f", {rest} {'needs' if rest == 1 else 'need'} the internet " + f"({offline_named()})" if rest else ""), "title") + if not rest: + # The remedy was for files of the whole list: none of the + # choice needs the internet. + names = [_last_name(str(folder)) for folder in state.from_folders] + remedies = {_offline_remedy(names), _offline_remedy(names, named=True)} + lines = [line for line in lines if str(line) not in remedies] # The in place and missing counts are the whole list's: under an # estimate for some systems they read as its own. whole = (f"{check['present']} in place, " @@ -18472,6 +18655,14 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: shown += 1 if shown > 20: lines.append(f" ... and {shown - 20} more") + expected = (state.dry if state.dry is not None else state.check or {}).get( + "data_missing", 0) + landed = max(0, installed - sum(1 for _dest, good, _why in state.finished if good)) + if expected > landed and not left and title != "Stopped" and summary: + # The data files the plan counted, never named on a result that + # counted the BIOS files alone (tui_audit_14 P3-1). + lines.append(f"{_plural(expected - landed, 'data file')} not installed" + + (f": not in {LOCAL.pack.name}." if LOCAL.pack is not None else ".")) if state.dest is not None and any(failure["reason"] == NOT_WRITABLE for failure in failed): # The plan's own advice: the result offered nothing after 22 files @@ -19057,6 +19248,7 @@ def tui_start( captured = io.StringIO() with contextlib.redirect_stdout(captured), contextlib.redirect_stderr(captured): state.setups, state.mounted_setups = found_setups(os_type) + state.unnamed = unnamed_cards() frontends = detect_frontends(os_type) # Said on the normal screen before this one covered it, and kept # above the list: a status line under 18 setups was never drawn. diff --git a/install.sh b/install.sh index d96b9683c..6b3f40947 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,12 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="3f450bbd97735e20ce25bd4a7d89e94994d6162edaec108a63f8861452143d2c" +# What a refused installer most likely means for this copy. The copy the +# site serves reads a commit's installer, which no cache serves stale, and +# curl or wget stop on a cut transfer before the hash is read: the site's +# build rewrites this line there (scripts/sitebootstrap.py). +MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." +DEFAULT_INSTALL_SHA256="48a5fca25303757fc2874262c1d176992bd7340da5c59f58c6e911383bf54817" MAX_INSTALLER_BYTES=2097152 cleanup() { @@ -110,9 +115,7 @@ if [ -z "$INSTALLER" ] || [ ! -f "$INSTALLER" ]; then echo " accepted: unset it to use the installer this script names, or set it to match." >&2 exit 1 fi - echo " The download was most likely cut short: try again. Within minutes of an update" >&2 - echo " of this project the two files can come from caches of different ages." >&2 - echo " Waiting five minutes settles that." >&2 + echo " $MISMATCH_ADVICE" >&2 exit 1 fi INSTALLER="$TEMP_INSTALLER" diff --git a/install/launchers/index.json b/install/launchers/index.json index c3e65b9c5..3fe893f19 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 919079, - "sha256": "3f450bbd97735e20ce25bd4a7d89e94994d6162edaec108a63f8861452143d2c", + "size": 929922, + "sha256": "48a5fca25303757fc2874262c1d176992bd7340da5c59f58c6e911383bf54817", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 972256, - "sha256": "23a6a602e3478929b2d3ad01e600ff9a72343dc764cae2ea29dbcd59c790c685" + "size": 983099, + "sha256": "78658671fe962a5a4efb8ea4a3a50960cc5680b5109391e8536951b5f0a97d14" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 919079, - "sha256": "3f450bbd97735e20ce25bd4a7d89e94994d6162edaec108a63f8861452143d2c", + "size": 929922, + "sha256": "48a5fca25303757fc2874262c1d176992bd7340da5c59f58c6e911383bf54817", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 919079, - "sha256": "3f450bbd97735e20ce25bd4a7d89e94994d6162edaec108a63f8861452143d2c", + "size": 929922, + "sha256": "48a5fca25303757fc2874262c1d176992bd7340da5c59f58c6e911383bf54817", "executable": false }, { diff --git a/scripts/sitebootstrap.py b/scripts/sitebootstrap.py index 05d80e22f..6856d86f1 100644 --- a/scripts/sitebootstrap.py +++ b/scripts/sitebootstrap.py @@ -15,6 +15,7 @@ Invoke-RestMethod reads any type that is neither JSON nor XML as text. from __future__ import annotations +import hashlib import os import re import subprocess @@ -32,6 +33,22 @@ SOURCES: dict[str, tuple[str, re.Pattern[str]]] = { SH_NAME: ("install.sh", re.compile(r'^(DEFAULT_INSTALL_URL=")[^"]*(")', re.M)), PS1_NAME: ("install.ps1", re.compile(r'^(\s*\$defaultInstallUrl = ")[^"]*(")', re.M)), } +# The line each bootstrap says a refused installer with, and the pin. +ADVICE: dict[str, re.Pattern[str]] = { + SH_NAME: re.compile(r'^(MISMATCH_ADVICE=")[^"]*(")', re.M), + PS1_NAME: re.compile(r'^(\s*\$mismatchAdvice = ")[^"]*(")', re.M), +} +PIN: dict[str, re.Pattern[str]] = { + SH_NAME: re.compile(r'^DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', re.M), + PS1_NAME: re.compile(r'^\s*\$defaultInstallSha256 = "([0-9a-f]{64})"', re.M), +} +# What the served copy says of a refused installer. It names a commit's +# install.py, which never changes, and both downloaders fail on a short +# transfer before the hash is read: neither a cut download nor caches of +# two ages explain a mismatch there (code_audit_21 P3-1). +SERVED_ADVICE = ("The file was changed on its way here, most often by a proxy or an " + "antivirus that rewrites downloads. Run the command again from another " + "network, or with that web protection paused.") def build_commit(root: Path) -> str: @@ -52,14 +69,30 @@ def installer_url(commit: str) -> str: def served_text(name: str, root: Path, commit: str) -> str: - """The bootstrap served as name, its installer read from commit.""" + """The bootstrap served as name, its installer read from commit and its + advice fitted to that. A pin that is not the SHA-256 of root's + install.py stops the build: published, that copy refused every install + until the next good deploy, where a failed build keeps the last good + pair served (code_audit_21 P3-2).""" source, line = SOURCES[name] text = (root / source).read_text(encoding="utf-8") + pinned = PIN[name].findall(text) + if len(pinned) != 1: + raise ValueError(f"{source}: {len(pinned)} installer pins, expected one") + actual = hashlib.sha256((root / "install.py").read_bytes()).hexdigest() + if pinned[0] != actual: + raise ValueError(f"{source} pins {pinned[0][:12]}, install.py is {actual[:12]}: " + "re-pin the bootstrap before the site serves it") served, count = line.subn( lambda m: f"{m.group(1)}{installer_url(commit)}{m.group(2)}", text ) if count != 1: raise ValueError(f"{source}: {count} installer address lines, expected one") + served, count = ADVICE[name].subn( + lambda m: f"{m.group(1)}{SERVED_ADVICE}{m.group(2)}", served + ) + if count != 1: + raise ValueError(f"{source}: {count} mismatch advice lines, expected one") return served diff --git a/tests/test_install.py b/tests/test_install.py index 5488c4fa9..991fb37af 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -4210,8 +4210,11 @@ class RunsAgainstTheRealLists(unittest.TestCase, _LoopbackRun): self.assertEqual(proc.returncode, 0, proc.stderr) lines = [line for line in proc.stdout.splitlines() if line.strip()] self.assertIn("Nothing was written", lines[-1]) - header = next(line for line in lines if line.startswith("Would download")) - self.assertIn("data files", header) + at = next(index for index, line in enumerate(lines) if line.startswith("Would download")) + # One total on the header, the data files among the kinds under it + # (code_audit_21 P3-8). + self.assertNotIn("data files", lines[at]) + self.assertIn("data files", lines[at + 1]) def test_dest_alone_says_whose_list_it_is(self): proc = _install_run(self.env, "--dest", str(self.home / "card"), "--dry-run", diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 521ceccbc..a0dfb7a48 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -9203,7 +9203,7 @@ class AuditSixteenAndNineLineMode(unittest.TestCase): folder = install.ask_platform_folder("linux", "batocera") self.assertEqual(folder, share / "bios") text = out.getvalue() - self.assertIn(f"1) SHARE: bios ({share / 'bios'}, created)", text) + self.assertIn(f"1) SHARE: bios ({share / 'bios'}, does not exist yet)", text) self.assertNotIn("MUOS", text) def test_the_folder_question_quits_on_q_and_says_what_to_type(self): @@ -12215,6 +12215,176 @@ class AuditTwentyOneAndFourteen(unittest.TestCase): install.tui_update(state, Key("enter")) self.assertEqual(state.exit_code, 0) + def test_offline_the_plan_counts_the_systems_ticked(self): + """code_audit_21 P3-5: offline, one system ticked, the plan counted + every file of the list beside "For your 1 of 3 systems".""" + state = checked_state() + folder = str(self.root / "from1") + state.from_folders = [self.root / "from1"] + state.found = [found("gba_bios.bin", folder)] + with mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}): + state.ticked = {"nintendo-gba"} + text = self.screen(state, 100, 40) + self.assertIn("About 1 file, 16.0 KB, for 1 of 3 systems ", text) + self.assertNotIn("found in from1", text) + self.assertNotIn("Run the installer again", text) + state.ticked = {"sony-playstation"} + text = self.screen(state, 100, 40) + self.assertIn("About 0 files, 0 B, for 1 of 3 systems, 3 need the internet", text) + + def easyroms(self, name: str = "EASYROMS") -> Path: + """An ArkOS games part as a computer mounts it: bios/ and console + folders, no file naming the system.""" + card = self.root / "media" / name + for folder in ("bios", "nes", "gba", "psx"): + (card / folder).mkdir(parents=True) + return card + + def test_an_arkos_card_is_found_and_comes_first(self): + """tui_audit_14 P2-3: an EASYROMS partition mounted, "Nothing was + found here", and two presses of Enter filled this computer's + ~/.config/retroarch/system for a RetroArch that was not there.""" + for name in ("EASYROMS", "GAMES"): + with self.subTest(card=name): + card = self.easyroms(name) + home = self.root / "home" + with mock.patch.object(install, "_mounted_cards", return_value=[card]), \ + mock.patch.object(install, "detect_platforms", return_value=[]), \ + mock.patch.object(install, "detect_frontends", return_value=[]), \ + mock.patch.object(install.Path, "home", return_value=home): + state, _effects = install.tui_start(_args(), "linux") + self.assertEqual(state.unnamed, [card]) + text = self.screen(state) + self.assertIn(f"A card is in the reader ({name}): choose its system", + text) + self.assertNotIn("Nothing was found here", text) + install.tui_update(state, Key("enter")) + self.assertEqual(state.screen, "folder") + rows = install.tui_rows(state) + self.assertEqual(rows[0].value, card / "bios") + install.tui_update(state, Key("enter")) + self.assertEqual(state.dest, card / "bios") + shutil.rmtree(card) + + def lacking_pack(self, held: list[tuple[str, bytes]]) -> Path: + pack_dir = self.root / "downloads" + pack_dir.mkdir(exist_ok=True) + pack = pack_dir / "RetroArch_BIOS_Pack.zip" + with zipfile.ZipFile(pack, "w") as archive: + archive.writestr("manifest.json", json.dumps({"files": []})) + for dest, payload in held: + archive.writestr(dest, payload) + return pack + + def test_line_mode_names_what_the_pack_lacks_before_its_plan(self): + """code_audit_21 P3-3: --dry-run promised 23 files, --check said to + fetch them, the install counted 23 beside the size of 20 and advised + retrying files that always fail.""" + files = [self.entry(f"bios{n}.bin", b"x" * (n + 1), ["nintendo-gba"]) + for n in range(4)] + listing = self.listing("retroarch", files) + pack = self.lacking_pack([("bios0.bin", b"x")]) + dest = self.root / "card" / "system" + common = dict(manifest=listing, from_pack=[pack], dest=dest, declared_only=False, + no_input=True) + text, _asked = self.line_run(_args(**common, dry_run=True), []) + lacks = "3 of these 4 files are not in RetroArch_BIOS_Pack.zip" + self.assertIn(lacks, text) + self.assertLess(text.index(lacks), text.index("Would copy")) + self.assertIn("Would copy 1 file (1 B) from the pack", text) + text, _asked = self.line_run(_args(**common, check=True), []) + self.assertIn(lacks, text) + self.assertNotIn("to fetch them", text) + self.assertIn("to copy 1 of them from RetroArch_BIOS_Pack.zip", text) + options = ("--platform", "retroarch", "--manifest", str(listing), "--dest", str(dest), + "--yes") + proc = install_run(*options, "--from-pack", str(pack)) + text = proc.stdout + proc.stderr + self.assertIn("Copying 1 file (1 B) from RetroArch_BIOS_Pack.zip...", text) + self.assertNotIn("retry only those", text) + self.assertIn("RetroArch_BIOS_Pack.zip does not hold them: a newer pack", text) + # A pack holding none: no copy, no folder. + shutil.rmtree(dest.parent, ignore_errors=True) + proc = install_run(*options, "--from-pack", str(self.lacking_pack([]))) + text = proc.stdout + proc.stderr + self.assertNotIn("Copying 0 files", text) + self.assertIn("RetroArch_BIOS_Pack.zip does not hold them", text) + self.assertFalse(dest.exists()) + + def test_a_pack_holding_none_offers_no_install_with_data_files(self): + """tui_audit_14 P3-1: data files left, the pack holding none of the + files nor the trees: Install was offered and failed every file, and + the result never named the data files.""" + state = checked_state() + state.check = {**state.check, "data_missing": 7, "data_bytes": 2048} + state.manifest = {**MANIFEST, "data_directories": [ + {"ref": "dolphin", "dest": "dolphin-emu/Sys", "files": {}}]} + + class Empty: + name = "RetroArch_BIOS_Pack.zip" + + @staticmethod + def find(entry: dict) -> None: + return None + + @staticmethod + def tree(prefix: str): + return iter(()) + + with mock.patch.object(install.LOCAL, "pack", Empty()): + actions = [row.action for row in install.tui_rows(state)] + self.assertNotIn("install", actions) + self.assertEqual(actions[0], "fromcard") + text = self.screen(state) + self.assertIn("RetroArch_BIOS_Pack.zip holds none of these 11 files", text) + # The result of an install run anyway names them. + state.stack = ["result"] + state.summary = {"installed": 0, "up_to_date": 0, "pending": 4, "kept": [], + "failed": [{"dest": event["dest"], "reason": "not in the pack"} + for event in state.files]} + state.code = 2 + _title, lines = install.tui_outcome(state) + self.assertIn("7 data files not installed: not in RetroArch_BIOS_Pack.zip.", lines) + + def test_a_missing_folder_does_not_exist_yet_on_every_row(self): + """code_audit_21 P3-7: line mode said "(created)" of folders nobody + created, and the screen said "does not exist yet" of the Windows row + alone.""" + drives = [self.root / "media" / "D", self.root / "media" / "E"] + for drive in drives: + drive.mkdir(parents=True) + (drives[1] / "bios").mkdir() + with mock.patch.object(install, "_mounted_cards", return_value=drives), \ + mock.patch.object(install, "_mount_points", return_value=frozenset()), \ + mock.patch.object(install, "_ask", return_value="q"), \ + contextlib.redirect_stdout(io.StringIO()) as out, \ + contextlib.suppress(SystemExit): + install.ask_platform_folder("linux", "retroarch") + text = out.getvalue() + self.assertNotIn(", created)", text) + self.assertNotIn("(created)", text) + self.assertIn(f"D: bios (card or drive) ({drives[0] / 'bios'}, does not exist yet)", text) + self.assertIn(f"E: bios (card or drive) ({drives[1] / 'bios'})\n", text) + state = fresh_state() + state.platform = "retroarch" + with mock.patch.object(install, "_mounted_cards", return_value=drives): + install._open_folder_screen(state, SYSTEM_DIR) + labels = [row.label for row in install.tui_rows(state) if row.action == "drive"] + self.assertEqual(labels, ["D: bios (card or drive) (does not exist yet)", + "E: bios (card or drive)"]) + + def test_line_mode_puts_the_arkos_card_first(self): + card = self.easyroms() + text, asked = self.line_run(_args(manifest=self.game_listing(), dry_run=True, + declared_only=False), ["1", ""], mounts=[card]) + self.assertIn("A card is in the reader (EASYROMS): choose its system.", text) + self.assertNotIn("Nothing was found here", text) + self.assertIn(f"The card in the reader keeps them in its bios folder: {card / 'bios'}.", + text) + self.assertIn(f"1) EASYROMS: bios (card or drive) ({card / 'bios'})", text) + self.assertIn("Enter for the card's bios folder", asked[-2]) + self.assertIn(f"BIOS folder: {card / 'bios'}", text) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_site_bootstrap.py b/tests/test_site_bootstrap.py index 1ebe30d07..4babf8b43 100644 --- a/tests/test_site_bootstrap.py +++ b/tests/test_site_bootstrap.py @@ -57,8 +57,12 @@ class ServedCopies(unittest.TestCase): served = sitebootstrap.served_text(name, REPO_ROOT, self.commit).splitlines() self.assertEqual(len(original), len(served)) changed = [(a, b) for a, b in zip(original, served) if a != b] - self.assertEqual(len(changed), 1, changed) - before, after = changed[0] + # The address, and the advice a refusal gives (code_audit_21 + # P3-1). + self.assertEqual(len(changed), 2, changed) + advice = [pair for pair in changed if sitebootstrap.SERVED_ADVICE in pair[1]] + self.assertEqual(len(advice), 1, changed) + before, after = next(pair for pair in changed if pair not in advice) self.assertIn("https://raw.githubusercontent.com/Abdess/retrobios/main/install.py", before) self.assertEqual(after, before.replace( @@ -94,6 +98,51 @@ class ServedCopies(unittest.TestCase): self.assertEqual(path.read_text(encoding="utf-8"), sitebootstrap.served_text(path.name, REPO_ROOT, self.commit)) + def test_the_served_copy_never_blames_a_cut_download_or_caches(self): + """code_audit_21 P3-1: the commit-pinned copy said "cut short, try + again" and "waiting five minutes settles that", neither of which can + happen to it.""" + for name in sitebootstrap.SOURCES: + with self.subTest(name=name): + text = sitebootstrap.served_text(name, REPO_ROOT, self.commit) + self.assertNotIn("cut short", text) + self.assertNotIn("five minutes", text) + self.assertIn(sitebootstrap.SERVED_ADVICE, text) + # The raw form keeps its advice: a branch's copy can be stale. + self.assertIn("cut short", (REPO_ROOT / "install.sh").read_text(encoding="utf-8")) + page = (REPO_ROOT / "wiki" / "first-install.md").read_text(encoding="utf-8") + answer = page.split("does not match its expected fingerprint.", 1)[1].split("###", 1)[0] + self.assertNotIn("cut short", answer) + self.assertIn("proxy or an antivirus", " ".join(answer.split())) + + def test_a_stale_pin_stops_the_build(self): + """code_audit_21 P3-2: a pin that is not the installer's was served, + and refused every install until the next good deploy.""" + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() + else None) as scratch: + root = Path(scratch) + for source in ("install.sh", "install.ps1"): + (root / source).write_text((REPO_ROOT / source).read_text(encoding="utf-8"), + encoding="utf-8") + (root / "install.py").write_text('print("not the pinned installer")\n', + encoding="utf-8") + for name in sitebootstrap.SOURCES: + with self.subTest(name=name), self.assertRaises(ValueError): + sitebootstrap.served_text(name, root, self.commit) + + def test_deploy_site_checks_the_copies_before_it_publishes(self): + import yaml # noqa: PLC0415 + + steps = yaml.safe_load((REPO_ROOT / ".github/workflows/deploy-site.yml").read_text( + encoding="utf-8"))["jobs"]["build"]["steps"] + runs = [str(step.get("run", "")) for step in steps] + uses = [str(step.get("uses", "")) for step in steps] + check = next(i for i, run in enumerate(runs) if "tests.test_site_bootstrap" in run) + build = next(i for i, run in enumerate(runs) if "scripts/generate_site.py" in run) + upload = next(i for i, use in enumerate(uses) if "upload-pages-artifact" in use) + self.assertLess(build, check) + self.assertLess(check, upload) + def test_a_bootstrap_without_its_address_line_stops_the_build(self): with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None) as scratch: diff --git a/wiki/first-install.md b/wiki/first-install.md index eb08de448..37bd962c5 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -159,7 +159,9 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --platform retropie - ### Error: the downloaded installer does not match its expected fingerprint. -The download was cut short. Run the command again. +Nothing ran. The file was changed on its way, most often by a proxy or an +antivirus that rewrites downloads. Run the command from another network, or +with that web protection paused. ### Cannot create ... diff --git a/wiki/installer.md b/wiki/installer.md index 665c7313f..e506ba3f9 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1495,10 +1495,16 @@ pinned raw URL. A URL with any other scheme is refused outright. They cap the download at 2 MB and require the SHA-256 to equal the 64 hex characters embedded in the wrapper, `RETROBIOS_INSTALL_SHA256` when set. A mismatch aborts with "the downloaded installer does not match its expected -fingerprint" and nothing runs. Right after an update of this project the -wrapper and `install.py` can come from caches a few minutes apart, which reads -the same way: waiting five minutes settles it. Any change to `install.py` -therefore means recomputing that pin in both wrappers. +fingerprint" and nothing runs. The copies the site serves name the commit's +`install.py`, which never changes, and both downloaders stop on a cut transfer +before the hash is read: there a mismatch is a file changed on its way, by a +proxy or an antivirus, and that is what they say. The raw form reads `main`, +whose two files can come from caches a few minutes apart right after an +update: it says to try again, and that waiting five minutes settles it. The +site build refuses to publish a copy whose pin is not the SHA-256 of the +`install.py` it names, and Deploy Site runs the bootstrap tests before it +deploys, so a failed check keeps the last good pair served. Any change to +`install.py` therefore means recomputing that pin in both wrappers. They locate a Python 3.8 or newer interpreter (`python3` then `python` on POSIX, `py -3` then `python3` then `python` on Windows). Without one,