fix: games tools write whole files, name options

The games importers wrote through fixed .part names, the new options had no help text, and romfolders stopped on the first setup the offline cache lacked. RetroArch's rom folder map is regenerated from its source spec (path null, as the schema says). The installer's Python 3.8 job imports the games tests without pyyaml, so the region rules load lazily.
2026-11-06 01:53:56 -06:00 · 2026-11-03 22:46:17 +01:00
1 parent af2e930d32
commit fd7dd0efe5
@@ -1 +1 @@
-{"arcade":{"fbneo":{"file":"arcade/fbneo.json","gzip_size":115955,"sha256":"88961c9a6be804772baa767f6ee3a9162478c30c60fe55c8363d8fe4fbc909da","size":423362},"flycast":{"file":"arcade/flycast.json","gzip_size":1841,"sha256":"957189a6e993fda08e21c90ffc6ad94aed9ca9e6dd9ed8c7724e29db036791cb","size":5441},"mame":{"file":"arcade/mame.json","gzip_size":154527,"sha256":"b12010301293a218ecfac45706d81a26c3ac0c27eebeca7409bc8f6ce0163e9e","size":604648},"mame0258":{"file":"arcade/mame0258.json","gzip_size":141151,"sha256":"7ecff1979ed2ac123ea3ba4825dde6e2c31906415165fabefc9af93fa48a6de4","size":556059},"mame0278":{"file":"arcade/mame0278.json","gzip_size":148595,"sha256":"7881e3516f451ec83e886aaf41ebac9d60f497e1f954f49749d52890d7955cbf","size":582797},"mame2000":{"file":"arcade/mame2000.json","gzip_size":8624,"sha256":"f0d69b0872ecfc2845eeb7304543e5b17c11db82060b5514c206b6962ed08263","size":23316},"mame2003":{"file":"arcade/mame2003.json","gzip_size":17456,"sha256":"3b44e2a92c80a8f104b7781fcc1c776d025ad2cf4986638efa3ea041baa9a56d","size":48929},"mame2003_plus":{"file":"arcade/mame2003_plus.json","gzip_size":21077,"sha256":"2857283556fa49c20d8160ed113465bf0c2e643ed0b8acd2d38266a78be8e25e","size":62373},"mame2010":{"file":"arcade/mame2010.json","gzip_size":30869,"sha256":"50b1b381b4b07463f69f1f4c512dd1c98a78d4d0e9c3aeaf49b7b1b007b42152","size":96528},"mame2015":{"file":"arcade/mame2015.json","gzip_size":86990,"sha256":"afeb8c32f651421f74c6de057b0c0b837acd14c533b5160bb31f4411e0afffec","size":360117},"mame2016":{"file":"arcade/mame2016.json","gzip_size":101249,"sha256":"1f75147977d9d082a38b8f8e47a02b301f3d6e6db97c9b9282f3e7f27f26ee47","size":406190}},"containers":{"7z":{"description":"Per-file CRCs and sizes sit in the header at the end of the archive, LZMA-compressed by default (kEncodedHeader). One LZMA decode of the header, independent of the archive size, gives them. Python's lzma is an optional module.","re_read":true,"signature":{"bytes_hex":"377abcaf271c","offset":0},"source_ref":"ip7z/7zip@b88c6bc0 DOC/7zFormat.txt:171-189, DOC/7zFormat.txt:318-334, CPP/7zip/Archive/7z/7zHandlerOut.cpp:859"},"cd-sector":{"description":"User data of a CD sector. In a 2048-byte image (.iso) sector N starts at N * 2048. In a raw 2352-byte image it starts at N * 2352 plus 16 for Mode 1 and plus 24 for Mode 2 Form 1 (12 sync bytes, a 4-byte header, and for Mode 2 an 8-byte subheader).","re_read":true,"sector_size":{"iso":2048,"raw":2352},"source_ref":"stenzek/duckstation@526b4a12 src/util/iso_reader.cpp:157-171, src/util/cd_image.h:28-34","user_data_offset":{"mode1":16,"mode2":24}},"chd-v5":{"description":"MAME compressed hunks, big-endian. The header holds sizes, codecs and hashes only, never the system or the region. The overall SHA-1 is what MAME software lists record. Neither SHA-1 equals a Redump track hash: chdman hashes 2448-byte frames (subcode included, zero-filled), pads each track to 4 frames and byte-swaps audio, all tracks in one stream.","fields":{"compressors":{"encoding":"fourcc4","length":16,"offset":16},"header_length":{"encoding":"u32be","length":4,"offset":8,"value":124},"hunk_bytes":{"encoding":"u32be","length":4,"offset":56},"logical_bytes":{"encoding":"u64be","length":8,"offset":32},"map_offset":{"encoding":"u64be","length":8,"offset":40},"metadata_offset":{"encoding":"u64be","length":8,"offset":48},"overall_sha1":{"encoding":"hex","length":20,"offset":84},"parent_sha1":{"encoding":"hex","length":20,"offset":104},"raw_sha1":{"encoding":"hex","length":20,"offset":64},"unit_bytes":{"encoding":"u32be","length":4,"offset":60},"version":{"encoding":"u32be","length":4,"offset":12,"value":5}},"magic":{"bytes":"MComprHD","offset":0},"media_tags":{"CHCD":"cd","CHGD":"gd-rom","CHGT":"gd-rom","CHT2":"cd","CHTR":"cd","DVD ":"dvd","GDDD":"hard-disk"},"metadata_entry":{"description":"16-byte entry header, payload follows. Linked list from metadata_offset.","flags":{"encoding":"u8","length":1,"offset":4},"length":{"encoding":"u24be","length":3,"offset":5},"next":{"encoding":"u64be","length":8,"offset":8},"tag":{"encoding":"ascii","length":4,"offset":0}},"re_read":true,"source_ref":"mamedev/mame@f34f0250 src/lib/util/chd.h:148-162, src/lib/util/chd.cpp:2021-2061, src/lib/util/chd.cpp:2822-2831, src/lib/util/chd.h:212-235, src/lib/util/chd.cpp:1709-1749, src/tools/chdman.cpp:443-445, src/tools/chdman.cpp:2195-2203, src/lib/util/cdrom.cpp:2669-2672, src/emu/romload.cpp:1211"},"gamecube-disc":{"description":"Where disc offset 0 sits inside each container Dolphin opens.","formats":{"ciso":{"disc_header_at":32768,"magic":"CISO"},"gcz":{"disc_header_at":null,"magic_u32le":2970337281,"note":"block 0 is zlib unless bit 63 of its pointer is set"},"iso":{"disc_header_at":0,"magic":null},"rvz":{"disc_header_at":88,"magic":"RVZ\u0001","note":"first 0x80 disc bytes copied into header 2"},"wbfs":{"disc_header_at":null,"magic":"WBFS","note":"disc offset 0 at wlba[0] << wbfs_sector_shift"},"wia":{"disc_header_at":88,"magic":"WIA\u0001","note":Line truncated
+{"arcade":{"fbneo":{"file":"arcade/fbneo.json","gzip_size":115955,"sha256":"88961c9a6be804772baa767f6ee3a9162478c30c60fe55c8363d8fe4fbc909da","size":423362},"flycast":{"file":"arcade/flycast.json","gzip_size":1841,"sha256":"957189a6e993fda08e21c90ffc6ad94aed9ca9e6dd9ed8c7724e29db036791cb","size":5441},"mame":{"file":"arcade/mame.json","gzip_size":154527,"sha256":"b12010301293a218ecfac45706d81a26c3ac0c27eebeca7409bc8f6ce0163e9e","size":604648},"mame0258":{"file":"arcade/mame0258.json","gzip_size":141151,"sha256":"7ecff1979ed2ac123ea3ba4825dde6e2c31906415165fabefc9af93fa48a6de4","size":556059},"mame0278":{"file":"arcade/mame0278.json","gzip_size":148595,"sha256":"7881e3516f451ec83e886aaf41ebac9d60f497e1f954f49749d52890d7955cbf","size":582797},"mame2000":{"file":"arcade/mame2000.json","gzip_size":8624,"sha256":"f0d69b0872ecfc2845eeb7304543e5b17c11db82060b5514c206b6962ed08263","size":23316},"mame2003":{"file":"arcade/mame2003.json","gzip_size":17456,"sha256":"3b44e2a92c80a8f104b7781fcc1c776d025ad2cf4986638efa3ea041baa9a56d","size":48929},"mame2003_plus":{"file":"arcade/mame2003_plus.json","gzip_size":21077,"sha256":"2857283556fa49c20d8160ed113465bf0c2e643ed0b8acd2d38266a78be8e25e","size":62373},"mame2010":{"file":"arcade/mame2010.json","gzip_size":30869,"sha256":"50b1b381b4b07463f69f1f4c512dd1c98a78d4d0e9c3aeaf49b7b1b007b42152","size":96528},"mame2015":{"file":"arcade/mame2015.json","gzip_size":86990,"sha256":"afeb8c32f651421f74c6de057b0c0b837acd14c533b5160bb31f4411e0afffec","size":360117},"mame2016":{"file":"arcade/mame2016.json","gzip_size":101249,"sha256":"1f75147977d9d082a38b8f8e47a02b301f3d6e6db97c9b9282f3e7f27f26ee47","size":406190}},"containers":{"7z":{"description":"Per-file CRCs and sizes sit in the header at the end of the archive, LZMA-compressed by default (kEncodedHeader). One LZMA decode of the header, independent of the archive size, gives them. Python's lzma is an optional module.","re_read":true,"signature":{"bytes_hex":"377abcaf271c","offset":0},"source_ref":"ip7z/7zip@b88c6bc0 DOC/7zFormat.txt:171-189, DOC/7zFormat.txt:318-334, CPP/7zip/Archive/7z/7zHandlerOut.cpp:859"},"cd-sector":{"description":"User data of a CD sector. In a 2048-byte image (.iso) sector N starts at N * 2048. In a raw 2352-byte image it starts at N * 2352 plus 16 for Mode 1 and plus 24 for Mode 2 Form 1 (12 sync bytes, a 4-byte header, and for Mode 2 an 8-byte subheader).","re_read":true,"sector_size":{"iso":2048,"raw":2352},"source_ref":"stenzek/duckstation@526b4a12 src/util/iso_reader.cpp:157-171, src/util/cd_image.h:28-34","user_data_offset":{"mode1":16,"mode2":24}},"chd-v5":{"description":"MAME compressed hunks, big-endian. The header holds sizes, codecs and hashes only, never the system or the region. The overall SHA-1 is what MAME software lists record. Neither SHA-1 equals a Redump track hash: chdman hashes 2448-byte frames (subcode included, zero-filled), pads each track to 4 frames and byte-swaps audio, all tracks in one stream.","fields":{"compressors":{"encoding":"fourcc4","length":16,"offset":16},"header_length":{"encoding":"u32be","length":4,"offset":8,"value":124},"hunk_bytes":{"encoding":"u32be","length":4,"offset":56},"logical_bytes":{"encoding":"u64be","length":8,"offset":32},"map_offset":{"encoding":"u64be","length":8,"offset":40},"metadata_offset":{"encoding":"u64be","length":8,"offset":48},"overall_sha1":{"encoding":"hex","length":20,"offset":84},"parent_sha1":{"encoding":"hex","length":20,"offset":104},"raw_sha1":{"encoding":"hex","length":20,"offset":64},"unit_bytes":{"encoding":"u32be","length":4,"offset":60},"version":{"encoding":"u32be","length":4,"offset":12,"value":5}},"magic":{"bytes":"MComprHD","offset":0},"media_tags":{"CHCD":"cd","CHGD":"gd-rom","CHGT":"gd-rom","CHT2":"cd","CHTR":"cd","DVD ":"dvd","GDDD":"hard-disk"},"metadata_entry":{"description":"16-byte entry header, payload follows. Linked list from metadata_offset.","flags":{"encoding":"u8","length":1,"offset":4},"length":{"encoding":"u24be","length":3,"offset":5},"next":{"encoding":"u64be","length":8,"offset":8},"tag":{"encoding":"ascii","length":4,"offset":0}},"re_read":true,"source_ref":"mamedev/mame@f34f0250 src/lib/util/chd.h:148-162, src/lib/util/chd.cpp:2021-2061, src/lib/util/chd.cpp:2822-2831, src/lib/util/chd.h:212-235, src/lib/util/chd.cpp:1709-1749, src/tools/chdman.cpp:443-445, src/tools/chdman.cpp:2195-2203, src/lib/util/cdrom.cpp:2669-2672, src/emu/romload.cpp:1211"},"gamecube-disc":{"description":"Where disc offset 0 sits inside each container Dolphin opens.","formats":{"ciso":{"disc_header_at":32768,"magic":"CISO"},"gcz":{"disc_header_at":null,"magic_u32le":2970337281,"note":"block 0 is zlib unless bit 63 of its pointer is set"},"iso":{"disc_header_at":0,"magic":null},"rvz":{"disc_header_at":88,"magic":"RVZ\u0001","note":"first 0x80 disc bytes copied into header 2"},"wbfs":{"disc_header_at":null,"magic":"WBFS","note":"disc offset 0 at wlba[0] << wbfs_sector_shift"},"wia":{"disc_header_at":88,"magic":"WIA\u0001","note":Line truncated
@@ -1 +1 @@
-{"case_sensitive":null,"folders":[],"format_version":1,"name":"RetroArch","notes":["RetroArch alone defines no rom folder. rgui_browser_directory starts empty and only a platform default fills it, which no desktop driver sets (os configuration.c:3126,3295-3298). The path none says so.","Content reaches a core through playlists built by scanning against the content databases (os tasks/task_database.c), or from the file browser with a core the user picks. The system can only come from the game itself or from a folder the user names."],"platform":"retroarch","roms_root":{"from_bios":null,"path":"none","source_ref":"os configuration.c:3126,3295-3298"},"setup":"retroarch","sources":{"os":{"commit":"69a4f0ea1e8aaf442ae4858f2e7f2b31a1776576","repo":"https://github.com/libretro/RetroArch","tag":"v1.22.2"}}}
+{"case_sensitive":null,"folders":[],"format_version":1,"name":"RetroArch","notes":["RetroArch alone defines no rom folder. rgui_browser_directory starts empty and only a platform default fills it, which no desktop driver sets (os configuration.c:3126,3295-3298).","Content reaches a core through playlists built by scanning against the content databases (os tasks/task_database.c), or from the file browser with a core the user picks. The system can only come from the game itself or from a folder the user names."],"platform":"retroarch","roms_root":{"from_bios":null,"path":null,"source_ref":"os configuration.c:3126,3295-3298"},"setup":"retroarch","sources":{"os":{"commit":"69a4f0ea1e8aaf442ae4858f2e7f2b31a1776576","repo":"https://github.com/libretro/RetroArch","tag":"v1.22.2"}}}
@@ -7,13 +7,13 @@ sources:
    commit: 69a4f0ea1e8aaf442ae4858f2e7f2b31a1776576
    tag: v1.22.2
 roms_root:
-  path: none
+  path: null
  from_bios: null
  source_ref: "os configuration.c:3126,3295-3298"
 case_sensitive: null
 notes:
 - "RetroArch alone defines no rom folder. rgui_browser_directory starts empty and only a platform default\
-  \ fills it, which no desktop driver sets (os configuration.c:3126,3295-3298). The path none says so."
+  \ fills it, which no desktop driver sets (os configuration.c:3126,3295-3298)."
 - "Content reaches a core through playlists built by scanning against the content databases (os tasks/task_database.c),\
  \ or from the file browser with a core the user picks. The system can only come from the game itself\
  \ or from a folder the user names."
@@ -7,6 +7,7 @@ it is only meaningful if the comparison ignores the hour."""
 from __future__ import annotations

 from collections.abc import Callable, Iterator
+from typing import BinaryIO
 import contextlib
 import json
 import os
@@ -159,6 +160,29 @@ def copy_file_atomic(source: str, path: str) -> None:
            os.unlink(scratch)
        raise

+
+def copy_stream_atomic(stream: BinaryIO, path: str) -> None:
+    """Stream a download into place whole or not at all.
+
+    A fixed scratch name let two runs interleave their bytes in one file of
+    the right size, which the rename then promoted."""
+    import shutil
+
+    directory = os.path.dirname(os.path.abspath(path))
+    handle, scratch = tempfile.mkstemp(
+        dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(handle, "wb") as f:
+            shutil.copyfileobj(stream, f)
+        set_plain_mode(scratch)
+        os.replace(scratch, path)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(scratch)
+        raise
+
+
 def _strip_timestamps(text: str) -> str:
    """Remove known timestamp patterns for content comparison."""
    result = text
@@ -237,10 +237,14 @@ def summary(regions: dict, extras: dict) -> dict:

 def main() -> int:
    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
-    parser.add_argument("--regions", type=Path, default=REGIONS_FILE)
-    parser.add_argument("--extras", type=Path, default=EXTRAS_FILE)
-    parser.add_argument("--headers", type=Path, default=HEADERS_FILE)
-    parser.add_argument("--emulators-dir", default="emulators")
+    parser.add_argument("--regions", type=Path, default=REGIONS_FILE,
+                        help=f"region rules to check (default {REGIONS_FILE})")
+    parser.add_argument("--extras", type=Path, default=EXTRAS_FILE,
+                        help=f"extra-file rules to check (default {EXTRAS_FILE})")
+    parser.add_argument("--headers", type=Path, default=HEADERS_FILE,
+                        help=f"header readers the triggers name (default {HEADERS_FILE})")
+    parser.add_argument("--emulators-dir", default="emulators",
+                        help="profiles the rules are checked against (default emulators)")
    parser.add_argument("--json", action="store_true", help="print the summary as JSON")
    args = parser.parse_args()

@@ -587,7 +587,8 @@ def print_coverage(coverage: dict, files: dict[Path, str]) -> None:

 def main() -> int:
    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
-    parser.add_argument("--output-dir", type=Path, default=OUTPUT)
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT,
+                        help=f"where the index is written (default {OUTPUT})")
    parser.add_argument("--check", action="store_true", help="exit 1 when a file would change")
    parser.add_argument("--quiet", action="store_true", help="print only failures")
    args = parser.parse_args()
@@ -49,6 +49,7 @@ from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path

+from ..artifacts import write_text_atomic
 from ..common import load_emulator_profiles
 from .romset_dat_importer import (
    LISTXML_CHUNK,
@@ -1117,21 +1118,24 @@ def write_if_changed(path: Path, text: str) -> bool:
    if path.is_file() and path.read_text(encoding="utf-8") == text:
        return False
    path.parent.mkdir(parents=True, exist_ok=True)
-    scratch = path.with_suffix(path.suffix + ".part")
-    scratch.write_text(text, encoding="utf-8")
-    scratch.replace(path)
+    write_text_atomic(str(path), text)
    return True


 def main() -> int:
    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
    which = parser.add_mutually_exclusive_group(required=True)
-    which.add_argument("--profile", action="append", choices=sorted(PROFILES))
-    which.add_argument("--all", action="store_true")
-    parser.add_argument("--emulators-dir", default="emulators")
-    parser.add_argument("--database", default="database.json")
-    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT)
-    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE)
+    which.add_argument("--profile", action="append", choices=sorted(PROFILES),
+                       help="profile to import (repeatable)")
+    which.add_argument("--all", action="store_true", help="every profile with an arcade source")
+    parser.add_argument("--emulators-dir", default="emulators",
+                        help="profiles whose BIOS sets are read (default emulators)")
+    parser.add_argument("--database", default="database.json",
+                        help="collection index, to name BIOS sets the collection lacks")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT,
+                        help=f"where the parent maps are written (default {DEFAULT_OUTPUT})")
+    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE,
+                        help=f"downloaded DATs and sources (default {DEFAULT_CACHE})")
    parser.add_argument("--offline", action="store_true", help="read the download cache only")
    parser.add_argument("--dry-run", action="store_true", help="print coverage, write nothing")
    args = parser.parse_args()
@@ -26,7 +26,6 @@ import gzip
 import io
 import json
 import re
-import shutil
 import sys
 import urllib.request
 import zipfile
@@ -39,6 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
 if _SCRIPTS_DIR not in sys.path:
    sys.path.insert(0, _SCRIPTS_DIR)

+from artifacts import copy_stream_atomic, write_text_atomic  # noqa: E402
 from common import parse_untrusted_xml  # noqa: E402
 from region import REGIONS, WORLD  # noqa: E402
 from region_audit import NOINTRO  # noqa: E402
@@ -279,9 +279,7 @@ def write_if_changed(path: Path, text: str) -> bool:
    if path.is_file() and path.read_text(encoding="utf-8") == text:
        return False
    path.parent.mkdir(parents=True, exist_ok=True)
-    scratch = path.with_suffix(path.suffix + ".part")
-    scratch.write_text(text, encoding="utf-8")
-    scratch.replace(path)
+    write_text_atomic(str(path), text)
    return True


@@ -296,15 +294,9 @@ def _download(url: str, destination: Path, headers: dict[str, str] | None = None
    """Stream url to destination and return the response headers kept."""
    destination.parent.mkdir(parents=True, exist_ok=True)
    request = urllib.request.Request(url, headers={"User-Agent": "retrobios", **(headers or {})})
-    scratch = destination.with_suffix(destination.suffix + ".part")
-    try:
-        with urllib.request.urlopen(request, timeout=300) as response, scratch.open("wb") as out:
-            shutil.copyfileobj(response, out)
-            kept = {"content_disposition": response.headers.get("Content-Disposition", "")}
-        scratch.replace(destination)
-    finally:
-        scratch.unlink(missing_ok=True)
-    return kept
+    with urllib.request.urlopen(request, timeout=300) as response:
+        copy_stream_atomic(response, str(destination))
+        return {"content_disposition": response.headers.get("Content-Disposition", "")}


 def _today() -> str:
@@ -340,7 +332,7 @@ def fetch_nointro(cache_dir: Path, offline: bool, refresh: bool) -> tuple[Path,
        "asset_size": asset["size"],
        "fetched": _today(),
    }
-    meta_path.write_text(json.dumps(meta, indent=1) + "\n", encoding="utf-8")
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
    return pack, meta


@@ -359,7 +351,7 @@ def fetch_redump(code: str, cache_dir: Path, offline: bool, refresh: bool) -> tu
    if not match:
        raise RuntimeError(f"{url} answered without a datfile name")
    meta = {"file": match["file"], "url": url, "fetched": _today()}
-    meta_path.write_text(json.dumps(meta, indent=1) + "\n", encoding="utf-8")
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
    return pack, meta


@@ -459,8 +451,10 @@ def main() -> int:
    parser.add_argument("--refresh", action="store_true", help="ask the servers again")
    parser.add_argument("--floor", action="store_true",
                        help="fail when a system holds fewer games than its stored floor")
-    parser.add_argument("--cache-dir", type=Path, default=CACHE_DIR)
-    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR)
+    parser.add_argument("--cache-dir", type=Path, default=CACHE_DIR,
+                        help=f"downloaded packs (default {CACHE_DIR})")
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR,
+                        help=f"where the catalogs are written (default {OUTPUT_DIR})")
    parser.add_argument("--dry-run", action="store_true", help="print coverage, write nothing")
    args = parser.parse_args()

@@ -14,6 +14,8 @@ from pathlib import Path
 import yaml

 from . import Fetcher, SourceError, load_sources, scrape_setup
+from upstream import UpstreamError  # noqa: E402  (the package puts scripts/ on sys.path)
+from artifacts import write_text_atomic  # noqa: E402
 from ..base_scraper import _PlatformDumper

 DEFAULT_DIR = Path("platforms") / "romfolders"
@@ -31,8 +33,10 @@ def main() -> int:
    which.add_argument("--setup", action="append", help="setup id (repeatable)")
    which.add_argument("--all", action="store_true", help="every setup of the sources file")
    which.add_argument("--list", action="store_true", help="list the setups and their format")
-    parser.add_argument("--sources", type=Path, default=DEFAULT_DIR / "_sources")
-    parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIR)
+    parser.add_argument("--sources", type=Path, default=DEFAULT_DIR / "_sources",
+                        help="the setups, their repositories and pinned revisions")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIR,
+                        help=f"where <setup>.yml is written (default {DEFAULT_DIR})")
    parser.add_argument("--offline", action="store_true", help="read the revision cache only")
    parser.add_argument("--dry-run", action="store_true", help="print counts, write nothing")
    args = parser.parse_args()
@@ -53,7 +57,7 @@ def main() -> int:
        fetcher = Fetcher(spec.get("sources", {}), offline=args.offline)
        try:
            doc = scrape_setup(setup, spec, fetcher)
-        except (SourceError, ValueError, KeyError, OSError) as exc:
+        except (SourceError, UpstreamError, ValueError, KeyError, OSError) as exc:
            print(f"FAIL {setup}: {exc}", file=sys.stderr)
            failed += 1
            continue
@@ -63,7 +67,7 @@ def main() -> int:
            target = args.output_dir / f"{setup}.yml"
            text = dump_setup(doc)
            if not target.exists() or target.read_text(encoding="utf-8") != text:
-                target.write_text(text, encoding="utf-8")
+                write_text_atomic(str(target), text)
    return 1 if failed else 0


@@ -27,7 +27,6 @@ import zlib
 from pathlib import Path
 from unittest import mock

-import yaml

 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT))
@@ -529,6 +528,12 @@ class KeepEveryRegionCases(unittest.TestCase):

    @classmethod
    def setUpClass(cls) -> None:
+        # The installer's own jobs (Python 3.8 with pyte alone) import this
+        # module for its card builders, and install.py needs no pyyaml.
+        try:
+            import yaml  # noqa: PLC0415
+        except ImportError as exc:
+            raise unittest.SkipTest("pyyaml reads games/regions.yml") from exc
        with open(REPO_ROOT / "games" / "regions.yml", encoding="utf-8") as handle:
            cls.rules = yaml.safe_load(handle)

@@ -75,6 +75,9 @@ class ScriptsNeedOnlyPyyaml(unittest.TestCase):
        local = {p.stem for p in scripts.rglob("*.py")} | {
            p.name for p in scripts.iterdir() if p.is_dir()
        } | {"scripts", "__future__"}
+        # install.py is the repository's own single file, stdlib only (its
+        # tests hold it to that): generate_games_index reads its narrowing.
+        local.add("install")
        allowed = {"yaml"}
        found = []
        for path in sorted(scripts.rglob("*.py")):
@@ -738,6 +738,9 @@ class EveryGitHubApiCallerSendsTheToken(unittest.TestCase):
            str(path.relative_to(scripts))
            for path in [*scripts.rglob("*.py"), *scripts.rglob("*.sh")]
            if "api.github.com" in (source := path.read_text(encoding="utf-8"))
+            # arcade_parent_importer names API URLs and sends them through
+            # romset_dat_importer._api_json, which holds the token.
+            and ("Request(" in source or "urlopen(" in source or path.suffix == ".sh")
            and "GITHUB_TOKEN" not in source
            and "github_headers(" not in source
            and "github_token(" not in source