diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 1850eb72..755482c6 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -3033,9 +3033,11 @@ def _main(held: contextlib.ExitStack) -> None: sys.exit(1) db = load_database(args.db) if not args.platform and not args.emulator: - lookup_hashes( + # A lookup that found nothing is an answer a script must see. + if lookup_hashes( hashes, db, args.bios_dir, args.emulators_dir, args.platforms_dir - ) + ): + sys.exit(1) return zip_contents = build_zip_contents_index(db) with _pack_output_lock(args.output_dir): diff --git a/scripts/packresolve.py b/scripts/packresolve.py index cf16c4bd..53d4bc97 100644 --- a/scripts/packresolve.py +++ b/scripts/packresolve.py @@ -124,8 +124,9 @@ def lookup_hashes( bios_dir: str, emulators_dir: str, platforms_dir: str, -) -> None: - """Print diagnostic info for each hash.""" +) -> int: + """Print diagnostic info for each hash; return how many were not found.""" + missing = 0 files_db = db.get("files", {}) by_md5 = db.get("indexes", {}).get("by_md5", {}) by_crc32 = db.get("indexes", {}).get("by_crc32", {}) @@ -142,6 +143,7 @@ def lookup_hashes( if not sha1 or sha1 not in files_db: print(f"\n{hash_type.upper()}: {hash_val}") print(" NOT FOUND in database") + missing += 1 continue entry = files_db[sha1] @@ -174,6 +176,9 @@ def lookup_hashes( except (KeyError, OSError): pass print(f" In repo: {'YES' if in_repo else 'NO'}") + if missing: + print(f"\n{missing} of {len(hashes)} hash(es) not in the database") + return missing def resolve_file( file_entry: dict, diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 72ee842c..e895d7d7 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4210,5 +4210,23 @@ class EmuDeckRunsItsEmulatorsStandalone(unittest.TestCase): self.assertIn('"standalone_cores": list(cores)', source) + +class DocumentedHashLookupsFindTheirFile(unittest.TestCase): + """The --from-md5 example was a test fixture hash: copied from the docs, + it answered NOT FOUND, which then exited 0.""" + + def test_every_documented_lookup_is_in_the_database(self): + db = json.loads((ROOT / "database.json").read_text(encoding="utf-8")) + by_md5 = db["indexes"]["by_md5"] + found = 0 + for page in sorted((ROOT / "wiki").glob("*.md")): + text = page.read_text(encoding="utf-8") + for md5 in re.findall(r"--from-md5 ([0-9a-f]{32})\b", text): + found += 1 + with self.subTest(page=page.name, md5=md5): + self.assertIn(md5, by_md5) + self.assertGreater(found, 0) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 5d29b65e..228fd170 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -2635,11 +2635,12 @@ class TestE2E(unittest.TestCase): } buf = io.StringIO() with contextlib.redirect_stdout(buf): - lookup_hashes([("md5", "md5abc")], db, "bios", "emulators", "platforms") + missing = lookup_hashes([("md5", "md5abc")], db, "bios", "emulators", "platforms") output = buf.getvalue() self.assertIn("test.bin", output) self.assertIn("sha1abc", output) self.assertIn("alt.bin", output) + self.assertEqual(missing, 0) def test_141_lookup_hashes_not_found(self): """lookup_hashes reports unknown hashes.""" @@ -2651,11 +2652,13 @@ class TestE2E(unittest.TestCase): db = {"files": {}, "indexes": {"by_md5": {}, "by_crc32": {}}} buf = io.StringIO() with contextlib.redirect_stdout(buf): - lookup_hashes( + missing = lookup_hashes( [("md5", "unknown123" + "0" * 22)], db, "bios", "emulators", "platforms" ) output = buf.getvalue() self.assertIn("NOT FOUND", output) + # The CLI exited 0 on this, so a script read "found". + self.assertEqual(missing, 1) def test_142_from_md5_platform_pack(self): """--from-md5 with --platform generates correctly laid out ZIP.""" diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index bfbeb44f..277e77e3 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -9,10 +9,15 @@ Fine-grained control over pack generation, hardware filtering, truth analysis, a Look up a single MD5 in the database: ```bash -python scripts/generate_pack.py --from-md5 d8f1206299c48946e6ec5ef96d014eaa +python scripts/generate_pack.py --from-md5 490f666e1afb15b7362b406ed1cea246 ``` -Build a pack containing only files matching hashes from a list (one MD5 per line, `#` for comments): +It prints the file's name, hashes, path and whether the collection holds it, +and exits 1 when a hash is not in the database, so a script can tell. + +Build a pack containing only files matching hashes from a list. Each line +carries one hash (MD5, SHA-1 or CRC32), alone or beside a file name as +`md5sum` writes it; `#` starts a comment: ```bash python scripts/generate_pack.py --platform batocera --from-md5-file missing.txt diff --git a/wiki/tools.md b/wiki/tools.md index 0a556e69..c13d5e9e 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -136,7 +136,7 @@ python scripts/generate_pack.py --platform retroarch --split python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer # Hash-based lookup and custom packs -python scripts/generate_pack.py --from-md5 d8f1206299c48946e6ec5ef96d014eaa +python scripts/generate_pack.py --from-md5 490f666e1afb15b7362b406ed1cea246 python scripts/generate_pack.py --platform batocera --from-md5-file missing.txt python scripts/generate_pack.py --platform retroarch --list-systems @@ -190,7 +190,9 @@ would have loaded. - `--split`: generate one ZIP per system instead of one big pack - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...) - `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator` -- `--from-md5-file`: same, reading hashes from a file (one per line, comments with #) +- `--from-md5-file`: same, reading hashes from a file: one MD5, SHA-1 or CRC32 + per line, alone or beside a file name as `md5sum` writes it, `#` for comments. + A lookup exits 1 when a hash is not in the database - `--target`: filter by hardware target (e.g. `switch`, `rpi4`, `x86_64`) - `--region`: ordered priority list, best first. Regions nest, a system with no regional split keeps everything, and a group with no candidate in any named