diff --git a/scripts/auto_fetch.py b/scripts/auto_fetch.py index 772667ad..f4fb3909 100644 --- a/scripts/auto_fetch.py +++ b/scripts/auto_fetch.py @@ -66,39 +66,51 @@ ARCHIVE_ORG_COLLECTIONS = [ def find_missing(config: dict, db: dict) -> list[dict]: - """Find BIOS files required by platform but not in database.""" - missing = [] + """Files the platform's own check or its cores' code would not find. + Read from verify, the platform's native check: a hash or name lookup + of our own called pc6001.zip and fmtowns.zip missing on Batocera, which + checks them inside the zip and holds them, and saw none of the files + the cores need that no platform lists. + """ + from nativemode import reads_file_contents # noqa: PLC0415 + from verify import Status, verify_platform # noqa: PLC0415 + + result = verify_platform(config, db) + hashed = reads_file_contents(config.get("verification_mode", "existence")) + wanted = { + (d.get("system", ""), d.get("name", "")) + for d in result["details"] + if d["status"] == Status.MISSING or (hashed and d["status"] == Status.UNTESTED) + } + missing: list[dict] = [] for sys_id, system in config.get("systems", {}).items(): for file_entry in system.get("files", []): - storage = file_entry.get("storage", "embedded") - if storage != "embedded": - continue - - sha1 = file_entry.get("sha1") - md5 = file_entry.get("md5") name = file_entry.get("name", "") - - found = False - if sha1 and sha1 in db.get("files", {}): - found = True - elif md5: - by_md5 = db.get("indexes", {}).get("by_md5", {}) - md5_list = [m.strip() for m in md5.split(",") if m.strip()] - found = any(m in by_md5 for m in md5_list) - - if not found: - missing.append( - { - "name": name, - "system": sys_id, - "sha1": sha1, - "md5": md5, - "size": file_entry.get("size"), - "destination": file_entry.get("destination", name), - } - ) - + if (sys_id, name) not in wanted: + continue + if file_entry.get("storage", "embedded") != "embedded": + continue + missing.append({ + "name": name, + "system": sys_id, + "sha1": file_entry.get("sha1"), + "md5": file_entry.get("md5"), + "size": file_entry.get("size"), + "destination": file_entry.get("destination", name), + }) + for u in result["undeclared_files"]: + if u.get("in_repo") or not u.get("required"): + continue + missing.append({ + "name": u["name"], + "system": u.get("system") or "", + "sha1": u.get("sha1"), + "md5": u.get("md5"), + "size": None, + "destination": u.get("path") or u["name"], + "emulator": u.get("emulator", ""), + }) return missing diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 0152c9f8..bd602b32 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3393,5 +3393,38 @@ class TheWantedPageCountsRomsetMembers(unittest.TestCase): self.assertEqual(len(calls), 1) + +class AutoFetchAsksVerifyWhatIsMissing(unittest.TestCase): + """auto_fetch judged presence by sha1 or md5 in the database: Batocera's + pc6001.zip, checked inside the zip and held, was a target, and no core + file was ever one. 721 announced where verify saw none critical.""" + + def test_the_targets_are_verify_verdicts(self): + from scripts import auto_fetch + import verify as verify_mod + + config = {"verification_mode": "md5", "systems": {"s": {"files": [ + {"name": "ok.bin", "md5": "1" * 32}, + {"name": "gone.bin", "md5": "2" * 32}, + {"name": "bad.bin", "md5": "3" * 32}, + ]}}} + result = { + "details": [ + {"system": "s", "name": "ok.bin", "status": verify_mod.Status.OK}, + {"system": "s", "name": "gone.bin", "status": verify_mod.Status.MISSING}, + {"system": "s", "name": "bad.bin", "status": verify_mod.Status.UNTESTED}, + ], + "undeclared_files": [ + {"name": "diabdat.mpq", "in_repo": False, "required": True, + "system": "diablo", "emulator": "DevilutionX"}, + {"name": "held.bin", "in_repo": True, "required": True}, + {"name": "extra.bin", "in_repo": False, "required": False}, + ], + } + with mock.patch.object(verify_mod, "verify_platform", lambda c, d: result): + names = [m["name"] for m in auto_fetch.find_missing(config, {})] + self.assertEqual(names, ["gone.bin", "bad.bin", "diabdat.mpq"]) + + if __name__ == "__main__": unittest.main()