diff --git a/scripts/cleanup_disk.py b/scripts/cleanup_disk.py index 5c393739b..1e13da6b4 100644 --- a/scripts/cleanup_disk.py +++ b/scripts/cleanup_disk.py @@ -8,7 +8,8 @@ sessions create theirs. A worktree elsewhere on disk and the automation worktree (.cache/autorun/worktree) are never candidates. One is removed only when no process runs in or holds a file of it, it is older than --hours, and nothing tracked is modified in it except generated artifacts (database.json, -install/*.json, exports/*.json, README.md, mkdocs.yml). Branches are always +install/*.json, exports/*.json, README.md, CONTRIBUTING.md, +website/src/data/*.json). Branches are always kept, so committed work stays reachable. A worktree with real uncommitted code is listed, never removed. @@ -39,7 +40,8 @@ from dataclasses import dataclass from pathlib import Path GENERATED = re.compile( - r"^(database\.json|install/.*\.json|exports/.*\.json|README\.md|mkdocs\.yml)$" + r"^(database\.json|install/.*\.json|exports/.*\.json|README\.md|CONTRIBUTING\.md" + r"|website/src/data/[^/]+\.json)$" ) KEPT_TMP = ("acquire",) AUTOMATION_WORKTREE = Path(".cache") / "autorun" / "worktree" diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 684c4b4a7..0847da15f 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -678,7 +678,8 @@ def main() -> None: "--skip-packs", action="store_true", help="Skip steps 4 to 6 except 4d: packs, manifests and their checks. " - "Steps 7 and 8 still rewrite README.md and docs/, add --skip-docs to " + "Steps 7 and 8 still rewrite README.md and the site pages in website/, " + "add --skip-docs to " "stop after verify", ) parser.add_argument( @@ -734,7 +735,8 @@ def main() -> None: # hold lasts until this process exits: taken step by step, it let another # run purge the packs between the build and the integrity check, which # then reported every platform SKIP and passed. - # database.json, install/, README.md and docs/ sit at fixed paths, so + # database.json, install/, README.md, CONTRIBUTING.md and website/ sit at + # fixed paths (artifacts.REPOSITORY_OUTPUTS), so # every run holds them too: a run on a scratch --output-dir otherwise # rewrote them beside the run on dist/. held = contextlib.ExitStack() diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 955b0dc52..fe6ff453a 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6576,7 +6576,8 @@ class SkipPacksHelpSaysWhatRuns(unittest.TestCase): ) text = " ".join(proc.stdout.split()) self.assertNotIn("Only regenerate DB and verify", text) - self.assertIn("Steps 7 and 8 still rewrite README.md and docs/", text) + self.assertIn("Steps 7 and 8 still rewrite README.md and the site pages in website/", text) + self.assertNotIn("docs/", text) def test_every_step_it_names_is_one_it_skips(self): sys.path.insert(0, str(ROOT / "scripts")) diff --git a/tests/test_autorun.py b/tests/test_autorun.py index fbdbf4912..354f64a7a 100644 --- a/tests/test_autorun.py +++ b/tests/test_autorun.py @@ -687,6 +687,15 @@ class Cleanup(Origin): def _decisions(self, busy=()) -> dict[str, cd.Decision]: return {d.path.name: d for d in cd.plan(self.repo, busy, self.HOURS, time.time())} + def test_generated_files_are_the_ones_a_run_rewrites(self): + """mkdocs.yml stayed in the list after the site moved to Starlight, + while CONTRIBUTING.md and website/src/data/*.json were left out.""" + for path in ("README.md", "CONTRIBUTING.md", "database.json", "install/retroarch.json", + "exports/batocera.json", "website/src/data/source-paths.json"): + self.assertTrue(cd.GENERATED.match(path), path) + for path in ("mkdocs.yml", "website/src/data/x/y.json", "scripts/verify.py"): + self.assertFalse(cd.GENERATED.match(path), path) + def test_worktree_and_tmp_rules(self): old_clean = self._worktree("old-clean", 100) old_code = self._worktree("old-code", 100) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index e3f88e212..5fb8d47a1 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1240,5 +1240,191 @@ class ExamplesComeFromTheData(unittest.TestCase): self.assertEqual(cited, entry["source_ref"]) + + +# A command a page gives: `python scripts/X.py ...` or `python -m scripts.x.y ...`, +# up to a comment, a pipe or the end of the line. A placeholder module +# (`myplatform_scraper`) is what a guide asks the reader to write. +_COMMAND = re.compile( + r"python3? (?:(scripts/[\w/]+\.py)|-m (scripts(?:\.\w+)+)(?![.*\w]))((?:[ \t]+[^\s#|&;`]+)*)" +) +_OPTION = re.compile(r"(? list[str]: + found = [] + for number, target, module, _args in _commands(text): + path = ROOT / (target.replace(".", "/") + ".py" if module else target) + if not path.is_file(): + found.append(f"{number}: {target}") + found += [ + f"{number}: {name}" + for number, line in enumerate(text.splitlines(), 1) + for name in re.findall(r"(? str: + key = (target, sub) + if key not in cls._helps: + import subprocess # noqa: PLC0415 + + argv = [sys.executable, *(["-m", target] if module else [target])] + proc = subprocess.run( + [*argv, *([sub] if sub else []), "--help"], + cwd=ROOT, capture_output=True, text=True, timeout=120, + ) + cls._helps[key] = proc.stdout if proc.returncode == 0 else "" + return cls._helps[key] + + @classmethod + def subcommands(cls, target: str, module: bool) -> list[str]: + usage = cls.help_text(target, module).split("\n\n", 1)[0] + choice = re.search(r"\{([\w,-]+)\}", usage) + return choice.group(1).split(",") if choice else [] + + @classmethod + def unknown(cls, text: str) -> list[str]: + found = [] + for number, target, module, args in _commands(text): + subs = cls.subcommands(target, module) + sub = args[0] if args and args[0] in subs else "" + known = set(_OPTION.findall(cls.help_text(target, module, sub))) + if not sub: + for other in subs: + if not other.startswith("_"): + known |= set(_OPTION.findall(cls.help_text(target, module, other))) + if not known: + found.append(f"{number}: {target} prints no --help") + continue + found += [ + f"{number}: {target} {sub} {option}".replace(" ", " ") + for arg in args + for option in _OPTION.findall(arg.split("=", 1)[0]) + if arg.startswith(option) and option not in known + ] + return found + + def test_every_cited_option_is_in_the_parser(self): + for page in sorted(WIKI.glob("*.md")): + with self.subTest(page=page.name): + self.assertEqual(self.unknown(page.read_text(encoding="utf-8")), []) + + def test_an_unknown_option_is_named(self): + self.assertEqual( + self.unknown("python scripts/campaign.py status --budget 6h\n"), + ["1: scripts/campaign.py status --budget"], + ) + + +class PipelineStepsMatchTheRun(unittest.TestCase): + """The step numbers the pages give are the ones pipeline.py prints. The + testing guide called the integrity check "step 6/8" in a run of 24 steps + numbered by label, and the tools table can lose or keep a step the run + no longer has.""" + + @staticmethod + def labels() -> list[str]: + import pipeline # noqa: PLC0415 + + return [pipeline.step_number(key) for key in pipeline.STEP_LABELS] + + @staticmethod + def cited(text: str) -> list[str]: + """Step numbers a sentence attributes to the pipeline.""" + found = [] + for match in re.finditer( + r"(?i)pipeline steps? ((?:\d[a-z0-9]*(?:/\d+)?(?:, | and | or )?)+)" + r"|steps? (\d[a-z0-9]*(?:/\d+)?) of the pipeline", + text, + ): + found += re.findall(r"\d[a-z0-9]*(?:/\d+)?", match.group(1) or match.group(2)) + return found + + def test_the_table_lists_every_step_in_order(self): + section = _section(_page("tools.md"), "Pipeline") + table = re.findall(r"^\| (\d[a-z0-9]*) \|", section, re.M) + self.assertEqual(table, self.labels()) + + def test_steps_named_in_the_pipeline_section_exist(self): + section = _section(_page("tools.md"), "Pipeline") + labels = set(self.labels()) + named = [ + step + for clause in re.findall(r"(?i)\bsteps? ((?:\(?\d[a-z0-9]*\)?(?:, | and |, and )?)+)", section) + for step in re.findall(r"\d[a-z0-9]*", clause) + ] + named += [step for group in re.findall(r"\(((?:\d[a-z0-9]*, )+\d[a-z0-9]*)\)", section) + for step in group.split(", ")] + self.assertTrue(named) + self.assertEqual(sorted(set(named) - labels), []) + + def test_every_page_cites_a_step_the_run_has(self): + labels = set(self.labels()) + for page in sorted(WIKI.glob("*.md")): + with self.subTest(page=page.name): + self.assertEqual( + [step for step in self.cited(page.read_text(encoding="utf-8")) + if step not in labels], []) + + def test_a_fraction_of_steps_is_refused(self): + self.assertEqual(self.cited("Integrated as pipeline step 6/8 (runs after"), ["6/8"]) + self.assertEqual(self.cited("pipeline steps 3a and 4d"), ["3a", "4d"]) + + +class SiteOutputIsTheStarlightProject(unittest.TestCase): + """generate_site.py writes into website/ since the move to Starlight, + while tools.md still gave `docs/` as the default of --docs-dir and as a + path the pipeline holds, and the testing guide measured pages in `docs/`.""" + + PAGES = ("tools.md", "testing-guide.md", "release-process.md", + "reading-reports.md", "measurements.md") + BARE_DOCS = re.compile(r"(?` on the platforms that get a pack, and reads its findings before cutting. +Before a release, a maintainer runs `python scripts/pack_probe.py run --platform +` on the released platforms it supports (RetroArch, Batocera and RetroBat), +and reads its findings before cutting. ## Which platforms get a pack { #which-platforms-get-a-pack } diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 6f2317e58..91d7585ac 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -257,9 +257,10 @@ in the platform YAML: Handles inner ZIP verification for MAME/FBNeo ROM sets (checkInsideZip, md5_composite, inner ROM MD5) and path collision deduplication. -One test per distinct pack contract: RetroArch, Batocera, BizHawk, EmuDeck, -Recalbox, RetroBat, RetroDECK, RomM. Inherited aliases such as Lakka reuse the -same artifact and are not counted twice. +One test walks every platform the registry knows, each a subtest, rather than +a list written by hand: eight names were once written out, and ROCKNIX and +MiSTer FPGA, both released, were never checked. A platform with no pack in +`dist/` is skipped, and so is a pack older than a profile it depends on. ```bash python -m unittest tests.test_pack_integrity -v @@ -267,8 +268,8 @@ python -m unittest tests.test_pack_integrity -v python scripts/generate_pack.py --all --verify-packs --output-dir dist/ ``` -Integrated as pipeline step 6/8 (runs after consistency check, before -README generation). Requires packs in `dist/`, skip with `--skip-packs`. +The pipeline runs the same check as step 6, after the consistency check and +before the README. It needs packs in `dist/`, and `--skip-packs` skips it. ### test_handheld_launchers.py @@ -385,7 +386,7 @@ the door open. The Start here pages and the guide part of each setup page are read by someone who knows nothing about BIOS files, often on a phone. Each word costs that reader attention, so `tests/test_site_copy.py` holds them to numbers measured -on the generated pages in `docs/`, code blocks left out: +on the generated pages in `website/src/content/docs/`, code blocks left out: | Budget | Start here | Setup guide | |--------|-----------:|------------:| @@ -412,7 +413,8 @@ a pack is a ZIP and a platform is a setup. The reasons: a reader holds about four things at once, scans more than reads, and reads less on a phone. Facts a newcomer does not act on (legal analysis, source lines, annotated runs) are kept, on the page one level down. To see -the numbers of every page, run `python -m tests.test_site_copy --report docs`. +the numbers of every page, run +`python -m tests.test_site_copy --report website/src/content/docs`. ### Adding a wiki page @@ -469,8 +471,11 @@ files it points at gives the installer a 404. The `validate.yml` workflow runs `python -m unittest discover -s tests -t . -v` on both roads into main: every pull request, and every direct push, that touches `bios/`, `platforms/`, `emulators/`, `schemas/`, `scripts/`, `tests/`, -`install.py` or an input of the site build such as `wiki/`. The same job then -builds the site with `npm run build --prefix website`, so a page that would fail the +`ports/`, the installer and its bootstraps, or an input of the site build such +as `wiki/` or `measurements/`. The `paths:` lists of `validate.yml` are the +reference, and `tests/test_workflow_paths.py` holds them to the site's inputs. The same job then +generates the site before the suite, since the tests read its pages, and builds +it with `npm run build --prefix website` after, so a page that would fail the deploy fails the pull request. Work reaches main by push as often as by pull request, so a suite wired to pull requests alone would guard the road nobody takes. The test job (`run-tests`) runs in parallel with schema validation, and on a pull request diff --git a/wiki/tools.md b/wiki/tools.md index 8a235e727..1aa6aef87 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -43,13 +43,13 @@ Pipeline steps: | 3 | Verify all platforms (`--strict`) | - | | 3a | File identity: one content under two identities (`identity.py --strict`, gates the run) | - | | 3b | Slot arbitration: destinations both layers claim (`slots.py`, reports only) | - | -| 4 | Generate packs | `--skip-packs` | +| 4 | Generate the packs of the platforms marked `release_pack` | `--skip-packs` | | 4b | Generate install manifests | `--skip-packs` | | 4c | Generate target manifests | `--skip-packs` | | 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - | | 4e | Games index: what the installer reads for the games on a card (`generate_games_index.py`, fails under its coverage floor) | - (reads the committed install lists) | -| 5 | Consistency check: counts, paths per archive, every emulator pack | if verify or pack skipped | -| 6 | Pack integrity (extract + hash) | `--skip-packs` | +| 5 | Consistency check: counts, paths per archive, every emulator pack | `--skip-packs`, or no full pack built (`--source platform` or `truth`) | +| 6 | Pack integrity (extract + hash) | `--skip-packs`, or no full pack built | | 7 | Generate README and CONTRIBUTING, then the installer page's tables and run samples (`generate_installer_doc.py`) | `--skip-docs` | | 8 | Generate site | `--skip-docs` | | 8b | Render the site with `npm run build --prefix website`, then `validate_site.py`, as the deploy does | `--skip-docs`, or Node not set up (`npm ci --prefix website`) | @@ -58,7 +58,7 @@ Pipeline flags: | Flag | Effect | |------|--------| -| `--offline` | skip every step that needs the network (2, 2a, 2a2, 2b, 2b2) | +| `--offline` | skip every step that needs the network (2, 2a, 2a2, 2b, 2b2), and pack only the release assets already in `.cache/large/` | | `--skip-packs` | skip steps 4, 4b, 4c, 6 and the consistency check. README and site are still rewritten unless `--skip-docs` is given too | | `--skip-docs` | skip steps 7 and 8 | | `--include-archived` | include archived platforms in verify, packs, truth and export | @@ -96,8 +96,8 @@ alone, run `python scripts/check_buildbot_system.py`. `/.lock`, and a second run on the same folder stops at once rather than read a half-written ZIP. Pack experiments with `generate_pack.py` use a scratch `--output-dir` under `tmp/`. A pipeline - run also writes `database.json`, `install/`, `README.md` and `docs/` at - fixed paths, so it holds the repository's `.lock` as well, and a second + run also writes `database.json`, `install/`, `README.md`, `CONTRIBUTING.md` + and `website/` at fixed paths, so it holds the repository's `.lock` as well, and a second pipeline is refused whatever its `--output-dir`. - **Read the exit status before the output.** A generator that crashed leaves the previous artifact in place, which reads exactly like a run that changed @@ -172,7 +172,7 @@ database or source tree, which is how the tests drive it against fixtures. Build platform-specific BIOS ZIP packs. ```bash -# Full platform packs +# Full platform packs (--all: the platforms marked release_pack) python scripts/generate_pack.py --all --output-dir dist/ python scripts/generate_pack.py --platform batocera python scripts/generate_pack.py --emulator dolphin @@ -643,6 +643,12 @@ python scripts/refresh_data_dirs.py --platform batocera # single platform onl python scripts/refresh_data_dirs.py --registry path/to/_data_dirs.yml ``` +A download that fails or does not read as an archive is tried three times, +after waits of 5 and 20 seconds: a mirror once answered with a page instead of +`ScummVM.zip` and served the archive minutes later. An archive that reads but +holds nothing under the expected path fails at once, since a retry gets the +same bytes. + ### check_freshness.py Ask every transcribed layer whether the local copy is the one upstream serves @@ -793,7 +799,7 @@ the linked worktrees placed inside the checkout, where sessions create theirs. One is removed only when no process uses it, it is older than `--hours`, and nothing tracked changed in it but generated files (`database.json`, `install/*.json`, `exports/*.json`, `README.md`, -`mkdocs.yml`). One holding real changes is listed, never removed. Branches +`CONTRIBUTING.md`, `website/src/data/*.json`). One holding real changes is listed, never removed. Branches are always kept. A worktree elsewhere on disk and the automation worktree are never candidates. `tmp/` entries older than `--hours` that no process uses are removed, except `tmp/acquire`, `*.bundle` and an entry that holds a @@ -953,30 +959,26 @@ python scripts/core_probe.py rejudge --dry-run python scripts/core_probe.py rejudge --emulator beetle_psx ``` +`python scripts/core_probe.py --help` lists every option. These need +more than its one line: + | Option | Effect | |--------|--------| -| `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST | -| `--content-label` | identity of the content, such as its Redump name, copied into the record | -| `--work-dir`, `--timeout` | scratch directory for the staged runs, and seconds allowed per repetition (default 1800), after which the run is killed and judged a hang | -| `--oracle FILE` | reference frames, error patterns and error screens the verdicts are judged against (default `tests/campaign/oracle.yml`) | +| `--timeout` | seconds allowed per repetition (default 1800), after which the run is killed and judged a hang | | `--repeat N` | runs of each variant, each in a fresh process (`run` defaults to 1, `check` to 2, `bench` to 5) | | `--vary DEST=SRC` | (`check`) one variant per occurrence. An empty SRC leaves DEST absent, `@zeros:SIZE` stages SIZE zero bytes and `@head:SIZE:PATH` the first SIZE bytes of PATH | | `--option KEY=VALUE` | core option. Unset options take the first value the core declares | -| `--frames`, `--warmup` | frames timed, and frames run before timing starts | | `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it | -| `--joypad-ports N` | ports given a joypad after loading, as RetroArch does (default 2) | | `--env KEY=VALUE` | environment of the core process. A core that reads the host clock repeats only with the clock fixed, for example `LD_PRELOAD` of a clock shim. `LD_PRELOAD` must name each library by an absolute path to a file, or the run is refused: the loader skips a library it cannot find without a word. A value naming a file is recorded by file name and sha256 | -| `--frame-hash-every K`, `--png-dir` | sha1 of every K-th frame, and those frames as PNG | -| `--state-var SECTION.VAR` | a variable of a Mednafen save state, as an observable | -| `--state-dir` | keep the save state taken at the end of the first repetition | -| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record | -| `--core-artifact URL` | where a downloaded `.so` came from, such as a buildbot nightly, copied into the record | +| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record. No core binary is kept in the repository: a record is reproduced by building the core again at that commit, so a nightly recorded without its commit cannot be | | `--record [DIR]` | append the record as one line of `measurements/.jsonl` in the checkout `DIR`, by default this one. `DIR` must be the root of a git checkout, never its `measurements/` folder. Commit it | | `--emulator` | the profile the record is filed under. By default the profile whose `cores:` names the `.so`, else the core's library name | | `--subject` | a slug naming the test, shared by its reruns. By default derived from the claim and method, or from the inputs of a run or bench | -| `--supersedes ID` | the id of an earlier record of the same emulator this one corrects | -| `--max-load` | refuse to record when the 1-minute load average is above it (default: a quarter of the CPU count) | -| `--no-trace` | run without strace | +| `--core-artifact URL` | where a downloaded `.so` came from, such as a buildbot nightly, copied into the record | +| `--joypad-ports N` | ports given a joypad after loading, as RetroArch does (default 2). | +| `--state-var SECTION.VAR`, `--state-dir` | a variable of a Mednafen save state as an observable, and a folder that keeps the state of the first repetition to read it again | +| `--max-load` | refuse to record above this 1-minute load average (default a quarter of the CPUs). Never raise it: a busy host computes wrong hashes | +| `--no-trace` | run without strace. The record then says that tracing was unavailable | | `@FILE` | read arguments from FILE, one per line | Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained), @@ -1164,14 +1166,12 @@ python scripts/region_check.py run \ python scripts/region_check.py report # the outcome of every region record, as JSON ``` -| Option | Effect | -|--------|--------| -| `--core-name` | the buildbot name of the core, whose x86_64 nightly is downloaded once into `.cache/region/cores/` and named in the record | -| `--core`, `--core-artifact` | a core `.so` to use instead, and where it was downloaded from | -| `--content-dir` | where romfetch keeps the test games (default `.cache/test-content`) | -| `--variants` | a subset of `full`, `own`, `others` and `pack` | -| `--timeout` | seconds per run (default 900) | -| `--max-load-wait` | seconds a run refused on a busy host is tried again, every 15 seconds, before it is given up (default 1800). The load threshold stays core_probe's | +`region_check.py run --help` lists the options. The nightly `--core-name` +names is downloaded once into `.cache/region/cores/`. `--core` and +`--core-artifact` run a core built elsewhere instead, `--content-dir` points +at another romfetch folder, and `--variants` reruns some of the four runs only. A run refused on a busy +host is tried again every 15 seconds for `--max-load-wait` seconds, and the +load threshold stays core_probe's. The file the core opened in the `full` run, read from strace, settles the claim: `confirmed` when its `region:` holds the game's region, `contradicted` @@ -1218,16 +1218,10 @@ python scripts/standalone_probe.py check pcsx2 \ | Files | `strace -f` inside the sandbox: the files the emulator opened in the staged BIOS folder, and the ones it looked for there and did not find | | Verdict | `core_probe.judge` on each repetition, with the oracle of `tests/campaign/oracle.yml` looked up by the content's sha1 and the emulator's name, plus the emulator's own error strings this script lists (DuckStation's "No BIOS image found for", PCSX2's "Unable to auto locate a BIOS image"). An emulator that exits before the last sample is a crash | -| Option | Effect | -|--------|--------| -| `--bios-file DEST=SRC` | copy SRC into the staged BIOS folder as DEST | -| `--setting SECTION.KEY=VALUE` | an emulator setting written before the run | -| `--sha256` | the fetched asset to run (default: the newest fetched) | -| `--samples`, `--interval` | screen samples per repetition (default 60), and seconds between two (default 1) | -| `--oracle` | the boot oracle file (default `tests/campaign/oracle.yml`, as `core_probe.py`) | -| `--png-dir` | each distinct sampled screen of the first repetition as PNG | -| `--record`, `--subject`, `--supersedes`, `--max-load`, `--work-dir`, `--no-trace`, `--require-sandbox` | as in `core_probe.py` | -| `check`: `--vary`, `--observable`, `--claim`, `--method`, `--repeat` | as in `core_probe.py`, compared on `verdict` by default | +`standalone_probe.py --help` lists the options. `--sha256` runs an +older fetched asset than the newest. The ones it shares with `core_probe.py` +(`--record`, `--max-load`, `--no-trace`, `--require-sandbox`) mean the same +there, and `check` compares on `verdict` by default. Supported: DuckStation and PCSX2. Dolphin, Flycast, PPSSPP, melonDS, xemu and MAME are installed by EmuDeck from Flathub, Ryujinx from a Codeberg tarball and @@ -1304,7 +1298,7 @@ before a release (wiki/release-process.md). Its lines stay in | `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | | `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | -| `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones, used by CI) | +| `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) | | `download.sh` | Same, as a shell one-liner (`curl` + `unzip`) | | `split_pack.py` | Cut a pack over the release asset limit into parts that are each a ZIP of whole files, each with a marker naming every part, read back before the pack is removed | @@ -1343,7 +1337,7 @@ Further options, by script: | `export_native.py` | `--truth-dir`, `--upstream-dir` (where the platforms' own files are cached) | | `find_pinned_revision.py` | `--repo URL --path FILE --md5 HASH`, `--write PATH`, `--cache-dir` | | `generate_db.py` | `--provenance-dir` (dump-catalog snapshots joined into the database) | -| `generate_site.py` | `--docs-dir` (output, `docs/` by default) | +| `generate_site.py` | `--site-dir` (the Starlight project, `website/` by default), `--docs-dir` (the pages, `src/content/docs` of that project by default) | | `mame_ref_audit.py` | `--write` points each stale ref at the line declaring its set, `--offline`, `--cache-dir` | | `migrate.py` | `--dry-run`, `--include-branches` also reads BIOS files from every remote branch | | `provenance_report.py` | `--provenance-dir`, `--write-derivable PATH` (with `--derive`, and the pipeline writes `derivable.json`) | @@ -1473,8 +1467,11 @@ release. `.gitignore` is the list of them: a `bios/` path listed there is a release asset, whatever its size, and a committed file is served from the repository even past 50 MB. `generate_db.py` downloads them from the release when rebuilding the database, -using `fetch_large_file()` from `common.py`. The same function is used by -`generate_pack.py` when a file has a hash mismatch with the local variant. +using `fetch_large_file()` from `largefiles.py`. `generate_pack.py` calls the +same function when the file is absent locally or holds another revision than +the entry declares, and keeps the asset only when it matches a declared hash. +The rules for asset names, uploads and checks are on the +[release process](release-process.md#large-files-management) page. ## Scrapers