fix: games tools write whole files, name options

The games importers wrote through fixed .part names, the new options had
no help text, and romfolders stopped on the first setup the offline cache
lacked. RetroArch's rom folder map is regenerated from its source spec
(path null, as the schema says). The installer's Python 3.8 job imports
the games tests without pyyaml, so the region rules load lazily.

scripts/scraper/arcade_parent_importer.py

@@ -49,6 +49,7 @@ from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path
 
+from ..artifacts import write_text_atomic
 from ..common import load_emulator_profiles
 from .romset_dat_importer import (
     LISTXML_CHUNK,
@@ -1117,21 +1118,24 @@ def write_if_changed(path: Path, text: str) -> bool:
     if path.is_file() and path.read_text(encoding="utf-8") == text:
         return False
     path.parent.mkdir(parents=True, exist_ok=True)
-    scratch = path.with_suffix(path.suffix + ".part")
-    scratch.write_text(text, encoding="utf-8")
-    scratch.replace(path)
+    write_text_atomic(str(path), text)
     return True
 
 
 def main() -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
     which = parser.add_mutually_exclusive_group(required=True)
-    which.add_argument("--profile", action="append", choices=sorted(PROFILES))
-    which.add_argument("--all", action="store_true")
-    parser.add_argument("--emulators-dir", default="emulators")
-    parser.add_argument("--database", default="database.json")
-    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT)
-    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE)
+    which.add_argument("--profile", action="append", choices=sorted(PROFILES),
+                       help="profile to import (repeatable)")
+    which.add_argument("--all", action="store_true", help="every profile with an arcade source")
+    parser.add_argument("--emulators-dir", default="emulators",
+                        help="profiles whose BIOS sets are read (default emulators)")
+    parser.add_argument("--database", default="database.json",
+                        help="collection index, to name BIOS sets the collection lacks")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT,
+                        help=f"where the parent maps are written (default {DEFAULT_OUTPUT})")
+    parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE,
+                        help=f"downloaded DATs and sources (default {DEFAULT_CACHE})")
     parser.add_argument("--offline", action="store_true", help="read the download cache only")
     parser.add_argument("--dry-run", action="store_true", help="print coverage, write nothing")
     args = parser.parse_args()

scripts/scraper/game_dat_importer.py

@@ -26,7 +26,6 @@ import gzip
 import io
 import json
 import re
-import shutil
 import sys
 import urllib.request
 import zipfile
@@ -39,6 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
 if _SCRIPTS_DIR not in sys.path:
     sys.path.insert(0, _SCRIPTS_DIR)
 
+from artifacts import copy_stream_atomic, write_text_atomic  # noqa: E402
 from common import parse_untrusted_xml  # noqa: E402
 from region import REGIONS, WORLD  # noqa: E402
 from region_audit import NOINTRO  # noqa: E402
@@ -279,9 +279,7 @@ def write_if_changed(path: Path, text: str) -> bool:
     if path.is_file() and path.read_text(encoding="utf-8") == text:
         return False
     path.parent.mkdir(parents=True, exist_ok=True)
-    scratch = path.with_suffix(path.suffix + ".part")
-    scratch.write_text(text, encoding="utf-8")
-    scratch.replace(path)
+    write_text_atomic(str(path), text)
     return True
 
 
@@ -296,15 +294,9 @@ def _download(url: str, destination: Path, headers: dict[str, str] | None = None
     """Stream url to destination and return the response headers kept."""
     destination.parent.mkdir(parents=True, exist_ok=True)
     request = urllib.request.Request(url, headers={"User-Agent": "retrobios", **(headers or {})})
-    scratch = destination.with_suffix(destination.suffix + ".part")
-    try:
-        with urllib.request.urlopen(request, timeout=300) as response, scratch.open("wb") as out:
-            shutil.copyfileobj(response, out)
-            kept = {"content_disposition": response.headers.get("Content-Disposition", "")}
-        scratch.replace(destination)
-    finally:
-        scratch.unlink(missing_ok=True)
-    return kept
+    with urllib.request.urlopen(request, timeout=300) as response:
+        copy_stream_atomic(response, str(destination))
+        return {"content_disposition": response.headers.get("Content-Disposition", "")}
 
 
 def _today() -> str:
@@ -340,7 +332,7 @@ def fetch_nointro(cache_dir: Path, offline: bool, refresh: bool) -> tuple[Path,
         "asset_size": asset["size"],
         "fetched": _today(),
     }
-    meta_path.write_text(json.dumps(meta, indent=1) + "\n", encoding="utf-8")
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
     return pack, meta
 
 
@@ -359,7 +351,7 @@ def fetch_redump(code: str, cache_dir: Path, offline: bool, refresh: bool) -> tu
     if not match:
         raise RuntimeError(f"{url} answered without a datfile name")
     meta = {"file": match["file"], "url": url, "fetched": _today()}
-    meta_path.write_text(json.dumps(meta, indent=1) + "\n", encoding="utf-8")
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
     return pack, meta
 
 
@@ -459,8 +451,10 @@ def main() -> int:
     parser.add_argument("--refresh", action="store_true", help="ask the servers again")
     parser.add_argument("--floor", action="store_true",
                         help="fail when a system holds fewer games than its stored floor")
-    parser.add_argument("--cache-dir", type=Path, default=CACHE_DIR)
-    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR)
+    parser.add_argument("--cache-dir", type=Path, default=CACHE_DIR,
+                        help=f"downloaded packs (default {CACHE_DIR})")
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR,
+                        help=f"where the catalogs are written (default {OUTPUT_DIR})")
     parser.add_argument("--dry-run", action="store_true", help="print coverage, write nothing")
     args = parser.parse_args()
 

scripts/scraper/romfolders/__main__.py

@@ -14,6 +14,8 @@ from pathlib import Path
 import yaml
 
 from . import Fetcher, SourceError, load_sources, scrape_setup
+from upstream import UpstreamError  # noqa: E402  (the package puts scripts/ on sys.path)
+from artifacts import write_text_atomic  # noqa: E402
 from ..base_scraper import _PlatformDumper
 
 DEFAULT_DIR = Path("platforms") / "romfolders"
@@ -31,8 +33,10 @@ def main() -> int:
     which.add_argument("--setup", action="append", help="setup id (repeatable)")
     which.add_argument("--all", action="store_true", help="every setup of the sources file")
     which.add_argument("--list", action="store_true", help="list the setups and their format")
-    parser.add_argument("--sources", type=Path, default=DEFAULT_DIR / "_sources")
-    parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIR)
+    parser.add_argument("--sources", type=Path, default=DEFAULT_DIR / "_sources",
+                        help="the setups, their repositories and pinned revisions")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIR,
+                        help=f"where <setup>.yml is written (default {DEFAULT_DIR})")
     parser.add_argument("--offline", action="store_true", help="read the revision cache only")
     parser.add_argument("--dry-run", action="store_true", help="print counts, write nothing")
     args = parser.parse_args()
@@ -53,7 +57,7 @@ def main() -> int:
         fetcher = Fetcher(spec.get("sources", {}), offline=args.offline)
         try:
             doc = scrape_setup(setup, spec, fetcher)
-        except (SourceError, ValueError, KeyError, OSError) as exc:
+        except (SourceError, UpstreamError, ValueError, KeyError, OSError) as exc:
             print(f"FAIL {setup}: {exc}", file=sys.stderr)
             failed += 1
             continue
@@ -63,7 +67,7 @@ def main() -> int:
             target = args.output_dir / f"{setup}.yml"
             text = dump_setup(doc)
             if not target.exists() or target.read_text(encoding="utf-8") != text:
-                target.write_text(text, encoding="utf-8")
+                write_text_atomic(str(target), text)
     return 1 if failed else 0