diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 8e1221ec2..960b2c3da 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -34,7 +34,9 @@ has ended, and its export is rewritten even when no row named it. Surfaced only (no refresh): data rows that are not a _data_dirs.yml key (the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs, -CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs +CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync, rom-folder +pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME +reads them again). Each needs a human read (new .info file, manual DAT download, PyPI version bump). Usage: @@ -65,7 +67,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml" DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml" AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs") -MANUAL_AREAS = ("coreinfo", "ci", "profiles") +# A rom-folder pin is moved by hand in platforms/romfolders/_sources/, +# then `python -m scripts.scraper.romfolders --setup NAME` reads it again. +MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles") JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10 @@ -563,7 +567,8 @@ def main() -> int: parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) parser.add_argument( "--only", - help="comma-separated areas to check: " + ",".join(AUTO_AREAS), + help="comma-separated areas to check: " + ",".join(AUTO_AREAS) + + " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)", ) parser.add_argument( "--dry-run", diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index dedd6629b..a1c463b0c 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -256,6 +256,16 @@ class SavedFindings(unittest.TestCase): [("catalogs", "mame recipes")]) self.assertEqual([f["subject"] for f in plan["surfaced"]], ["anybor"]) + def test_every_area_check_freshness_reports_is_known(self): + """romfolders was added to check_freshness and not here: its rows + were dropped from a saved report without a word, and --only + romfolders was refused.""" + import check_freshness # noqa: PLC0415 + + self.assertEqual(set(check_freshness.AREAS), set(rs.AUTO_AREAS + rs.MANUAL_AREAS)) + jobs, surfaced, _ = rs.plan_jobs([finding("romfolders", "muos")], REGISTRY, {}) + self.assertEqual((jobs, [f["subject"] for f in surfaced]), ([], ["muos"])) + def test_a_report_that_is_not_a_list_is_refused(self): with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory: path = Path(directory) / "freshness.json" diff --git a/wiki/tools.md b/wiki/tools.md index dab60e5b0..f566efe53 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -760,7 +760,10 @@ committed. Rows that need a decision are listed and never run: a buildbot core with no profile, a new `.info` file, the Redump, No-Intro and TOSEC packs, the CI -pins, and `profile_sync`. A failed scraper leaves its cached original +pins, `profile_sync`, and a rom-folder pin (moved by hand in +`platforms/romfolders/_sources/`, then `python -m scripts.scraper.romfolders +--setup NAME` reads it again). `--dry-run --json` prints the plan as JSON. +A failed scraper leaves its cached original untouched. Each job writes its output to `tmp/refresh_stale/`, and a GitHub token is taken from `GITHUB_TOKEN` or from `gh auth token`.