diff --git a/scripts/common.py b/scripts/common.py index b158f6bb..35d3a468 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1172,6 +1172,18 @@ def select_emulator_profiles( return selected +PROFILE_BACKUP_SUFFIX = ".old.yml" + + +def source_yaml_paths(directory: str | Path) -> list[Path]: + """YAML sources in a directory, without the copies the hash merge keeps.""" + return [ + path + for path in sorted(Path(directory).glob("*.yml")) + if not path.name.endswith(PROFILE_BACKUP_SUFFIX) + ] + + def load_emulator_profiles( emulators_dir: str, skip_aliases: bool = True, @@ -1199,9 +1211,7 @@ def load_emulator_profiles( file=sys.stderr, ) return profiles - for f in sorted(emu_path.glob("*.yml")): - if f.name.endswith(".old.yml"): - continue + for f in source_yaml_paths(emu_path): with open(f) as fh: profile = yaml_load(fh) or {} if "emulator" not in profile: diff --git a/scripts/generate_db.py b/scripts/generate_db.py index def852a6..2bb1a663 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -27,6 +27,7 @@ from common import ( list_registered_platforms, load_provenance_snapshots, require_yaml, + source_yaml_paths, write_if_changed, yaml_load, ) @@ -507,9 +508,7 @@ def _collect_all_aliases(files: dict) -> dict: # Collect aliases from emulator YAMLs (aliases field on file entries) emulators_dir = Path("emulators") if emulators_dir.is_dir(): - for emu_file in emulators_dir.glob("*.yml"): - if emu_file.name.endswith(".old.yml"): - continue + for emu_file in source_yaml_paths(emulators_dir): try: with open(emu_file) as f: emu_config = yaml_load(f) or {} diff --git a/scripts/scraper/_hash_merge.py b/scripts/scraper/_hash_merge.py index c78742d2..5ca29fa1 100644 --- a/scripts/scraper/_hash_merge.py +++ b/scripts/scraper/_hash_merge.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from common import write_text_atomic, yaml_load +from common import PROFILE_BACKUP_SUFFIX, write_text_atomic, yaml_load _MAME_RELEASE_RE = re.compile(r"^0\.\d+") @@ -381,7 +381,7 @@ def _backup_and_write(path: str, data: dict) -> None: (core_version, contents, source_ref), and appends new entries. """ p = Path(path) - backup = p.with_suffix(".old.yml") + backup = p.with_suffix(PROFILE_BACKUP_SUFFIX) shutil.copy2(p, backup) original = p.read_text(encoding="utf-8") @@ -572,7 +572,7 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None: Existing entries are left untouched (CRC32 changes are rare). """ p = Path(path) - backup = p.with_suffix(".old.yml") + backup = p.with_suffix(PROFILE_BACKUP_SUFFIX) shutil.copy2(p, backup) original = p.read_text(encoding="utf-8") diff --git a/scripts/scraper/fbneo_hash_scraper.py b/scripts/scraper/fbneo_hash_scraper.py index d52721db..e2a287a0 100644 --- a/scripts/scraper/fbneo_hash_scraper.py +++ b/scripts/scraper/fbneo_hash_scraper.py @@ -20,7 +20,7 @@ from typing import Any import yaml -from common import write_text_atomic, yaml_load +from common import source_yaml_paths, write_text_atomic, yaml_load from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile from scripts.scraper.fbneo_parser import parse_fbneo_source_tree @@ -200,9 +200,7 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]: def _find_fbneo_profiles() -> list[Path]: """Find emulator profiles whose upstream references finalburnneo/FBNeo.""" profiles: list[Path] = [] - for path in sorted(EMULATORS_DIR.glob("*.yml")): - if path.name.endswith(".old.yml"): - continue + for path in source_yaml_paths(EMULATORS_DIR): try: data = yaml_load(path.read_text(encoding="utf-8")) except (yaml.YAMLError, OSError): diff --git a/scripts/scraper/mame_hash_scraper.py b/scripts/scraper/mame_hash_scraper.py index ae51e6ec..0451e9bb 100644 --- a/scripts/scraper/mame_hash_scraper.py +++ b/scripts/scraper/mame_hash_scraper.py @@ -22,7 +22,7 @@ from typing import Any import yaml -from common import write_text_atomic, yaml_load +from common import source_yaml_paths, write_text_atomic, yaml_load from ._hash_merge import compute_diff, merge_mame_profile from .mame_parser import parse_mame_source_tree @@ -181,9 +181,7 @@ _MAME_TREES = frozenset({ def _find_mame_profiles() -> list[Path]: profiles: list[Path] = [] - for path in sorted(_EMULATORS_DIR.glob("*.yml")): - if path.name.endswith(".old.yml"): - continue + for path in source_yaml_paths(_EMULATORS_DIR): try: with open(path, encoding="utf-8") as f: data = yaml_load(f) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 1784e0fe..9ace88f2 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -12,7 +12,7 @@ from pathlib import Path, PurePosixPath import yaml -from common import yaml_load +from common import source_yaml_paths, yaml_load from jsonschema import Draft202012Validator, FormatChecker ROOT = Path(__file__).resolve().parent.parent @@ -46,7 +46,7 @@ def _validate_yaml_directory( ) -> list[str]: validator = _validator(schema_name) out: list[str] = [] - for path in sorted(directory.glob("*.yml")): + for path in source_yaml_paths(directory): if skip_private and path.name.startswith("_"): continue try: @@ -214,7 +214,7 @@ def _unknown_data_dir_refs() -> list[str]: registered = set((yaml_load(handle) or {}).get("data_directories", {})) out: list[str] = [] for directory in (ROOT / "emulators", ROOT / "platforms"): - for path in sorted(directory.glob("*.yml")): + for path in source_yaml_paths(directory): if path.name.startswith("_"): continue with path.open(encoding="utf-8") as handle: diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 136fdf6c..1d2c5411 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -2126,5 +2126,31 @@ class ModuleConstantsDeclaredOnce(unittest.TestCase): self.assertEqual(offenders, {}) +class HashMergeBackupsAreNotSources(unittest.TestCase): + """The hash merge copies a profile to .old.yml before rewriting it. + + Every loader skipped the copy except validate_schemas, which then failed + on a stale MAME profile nobody would ever ship. + """ + + def test_the_validator_skips_a_backup(self): + sys.path.insert(0, str(ROOT / "scripts")) + from validate_schemas import _validate_yaml_directory + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: + (Path(directory) / "mame.old.yml").write_text("emulator: [\n") + self.assertEqual( + _validate_yaml_directory(Path(directory), "emulator.schema.json"), [] + ) + + def test_the_backup_suffix_is_written_once(self): + offenders = [ + str(path.relative_to(ROOT)) + for path in sorted(ROOT.glob("scripts/**/*.py")) + if path.name != "common.py" and '".old.yml"' in path.read_text() + ] + self.assertEqual(offenders, []) + + if __name__ == "__main__": unittest.main()