docs: name both methods a scraper must define

This commit is contained in:
Abdessamad Derraz committed 2026-10-19 04:16:52 +02:00
1 parent 0af103542d
commit 4bfe4eef2c
2 files changed
+19 -3

No files matched your search

+11
View File
@@ -122,6 +122,17 @@ class DocumentedSteps(unittest.TestCase):
fields = [*schema["properties"], *item.get("properties", {})]
self.assertEqual(sorted(f for f in fields if f"`{f}`" not in page), [])
def test_the_scraper_recipe_names_every_abstract_method(self):
"""tools.md said implement fetch_requirements(); validate_format() is
abstract too, and a scraper written from the page could not be built."""
source = (SCRIPTS / "scraper" / "base_scraper.py").read_text(encoding="utf-8")
abstract = re.findall(r"@abstractmethod\s+def (\w+)", source)
self.assertTrue(abstract)
for page in ("tools.md", "adding-a-scraper.md"):
text = (REPO_ROOT / "wiki" / page).read_text(encoding="utf-8")
with self.subTest(page=page):
self.assertEqual([m for m in abstract if f"{m}(" not in text], [])
def test_every_function_the_pages_name_exists(self):
"""verification-modes described _find_best_variant(), deleted on
2026-10-05, and an algorithm the code no longer follows."""
+8 -3
View File
@@ -717,7 +717,10 @@ using `fetch_large_file()` from `common.py`. The same function is used by
## Scrapers
Located in `scripts/scraper/`. Each inherits `BaseScraper` and implements `fetch_requirements()`.
Located in `scripts/scraper/`. Each platform scraper inherits `BaseScraper` and
implements its two abstract methods, `fetch_requirements()` and
`validate_format()`; the MAME and FBNeo hash scrapers read a source tree and
stand alone.
| Scraper | Source | Format |
|---------|--------|--------|
@@ -743,8 +746,10 @@ and shared CLI), `dat_parser.py` (clrmamepro DAT format parser),
`fbneo_parser.py` (FBNeo C source BIOS set parser),
`_hash_merge.py` (text-based YAML patching that preserves formatting).
Adding a scraper: inherit `BaseScraper`, implement `fetch_requirements()`,
call `scraper_cli(YourScraper)` in `__main__`.
Adding a scraper: inherit `BaseScraper`, implement `fetch_requirements()` and
`validate_format()` (a class missing either cannot be instantiated), call
`scraper_cli(YourScraper)` in `__main__`. The full recipe is in
[adding-a-scraper.md](adding-a-scraper.md).
## Dump-catalog provenance