mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 13:33:24 -05:00
Abdessamad Derraz
6f27a296f7
kenji-nx cited tmp/es-de/ANDROID.md:470-474, a path from the machine of whoever profiled it. No revision of any declared repository holds it, so profile_sync could only report it missing, every pass, forever, and no amount of reading would ever settle it. validate_schemas now refuses a scratch directory, an absolute path, a Windows drive path and one climbing out of the tree, and names the offending citation rather than the scalar that carries it. Offline, so it runs on every push and every pull request rather than waiting for a network pass. The ES-DE citation reads as external now, which is what it always was. kenji-nx is at 37 refs, all anchored: the three changed blocks were var giving way to explicit types.
299 lines
11 KiB
Python
299 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate source and generated RetroBIOS data contracts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import yaml
|
|
|
|
from common import yaml_load
|
|
from jsonschema import Draft202012Validator, FormatChecker
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCHEMAS = ROOT / "schemas"
|
|
|
|
|
|
def _load_json(path: Path) -> object:
|
|
with path.open(encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
def _validator(name: str) -> Draft202012Validator:
|
|
schema = _load_json(SCHEMAS / name)
|
|
Draft202012Validator.check_schema(schema)
|
|
return Draft202012Validator(schema, format_checker=FormatChecker())
|
|
|
|
|
|
def _errors(validator: Draft202012Validator, data: object, label: str) -> list[str]:
|
|
out: list[str] = []
|
|
for error in sorted(validator.iter_errors(data), key=lambda item: list(item.path)):
|
|
location = "/".join(str(part) for part in error.absolute_path) or "<root>"
|
|
out.append(f"{label}:{location}: {error.message}")
|
|
return out
|
|
|
|
|
|
def _validate_yaml_directory(
|
|
directory: Path,
|
|
schema_name: str,
|
|
*,
|
|
skip_private: bool = False,
|
|
) -> list[str]:
|
|
validator = _validator(schema_name)
|
|
out: list[str] = []
|
|
for path in sorted(directory.glob("*.yml")):
|
|
if skip_private and path.name.startswith("_"):
|
|
continue
|
|
try:
|
|
with path.open(encoding="utf-8") as handle:
|
|
data = yaml_load(handle)
|
|
except (OSError, yaml.YAMLError) as exc:
|
|
out.append(f"{path.relative_to(ROOT)}: {exc}")
|
|
continue
|
|
out.extend(_errors(validator, data, str(path.relative_to(ROOT))))
|
|
out.extend(_unreachable_citations(path, data))
|
|
return out
|
|
|
|
|
|
def _validate_json_files(paths: list[Path], schema_name: str) -> list[str]:
|
|
validator = _validator(schema_name)
|
|
out: list[str] = []
|
|
for path in paths:
|
|
try:
|
|
data = _load_json(path)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
out.append(f"{path.relative_to(ROOT)}: {exc}")
|
|
continue
|
|
out.extend(_errors(validator, data, str(path.relative_to(ROOT))))
|
|
return out
|
|
|
|
|
|
def _validate_pack_manifests(dist: Path) -> list[str]:
|
|
"""Validate the integrity manifest each generated pack carries.
|
|
|
|
generate_pack.py writes manifest.json inside the archive, not beside it,
|
|
so a filesystem glob over dist/ matches nothing and silently validates
|
|
zero documents.
|
|
|
|
Reading a pack while a build is writing it reports "File is not a zip
|
|
file" about an archive that is merely half-written, so this takes the same
|
|
shared lock --verify-packs does. A build holding the exclusive lock means
|
|
the packs on disk are mid-flight and there is nothing stable to validate.
|
|
"""
|
|
if not dist.is_dir():
|
|
return []
|
|
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from common import ArtifactLockBusy, artifact_lock
|
|
|
|
try:
|
|
with artifact_lock(str(dist), exclusive=False):
|
|
return _scan_pack_manifests(dist)
|
|
except ArtifactLockBusy:
|
|
print(
|
|
f"note: {dist.name} is being written; skipping pack manifests",
|
|
file=sys.stderr,
|
|
)
|
|
return []
|
|
|
|
|
|
def _scan_pack_manifests(dist: Path) -> list[str]:
|
|
validator = _validator("pack-manifest.schema.json")
|
|
|
|
def _label(path: Path) -> str:
|
|
resolved = path.resolve()
|
|
try:
|
|
return str(resolved.relative_to(ROOT))
|
|
except ValueError:
|
|
return str(resolved)
|
|
|
|
out: list[str] = []
|
|
for archive in sorted(dist.glob("*.zip")):
|
|
try:
|
|
with zipfile.ZipFile(archive) as handle:
|
|
members = [
|
|
name
|
|
for name in handle.namelist()
|
|
if PurePosixPath(name).name == "manifest.json"
|
|
]
|
|
for member in members:
|
|
label = f"{_label(archive)}:{member}"
|
|
try:
|
|
document = json.loads(handle.read(member).decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
out.append(f"{label}: {exc}")
|
|
continue
|
|
out.extend(_errors(validator, document, label))
|
|
except (OSError, zipfile.BadZipFile) as exc:
|
|
out.append(f"{_label(archive)}: {exc}")
|
|
return out
|
|
|
|
|
|
_UNREACHABLE_REF = re.compile(
|
|
r"(?:^|[\s,;(])(?:tmp/|/|[A-Za-z]:[\\/]|\.{1,2}/)[\w.\\/+-]*[\w+-]:\d+(?:-\d+)?"
|
|
)
|
|
|
|
|
|
def _unreachable_citations(path: Path, document: object) -> list[str]:
|
|
"""Refs naming a place no repository can hold.
|
|
|
|
A citation is read against a revision of a declared repository, so a
|
|
scratch directory, an absolute path or one climbing out of the tree can
|
|
never resolve and never will. kenji-nx carried tmp/es-de/ANDROID.md:470,
|
|
a path from whoever profiled it, which profile_sync could only report
|
|
missing forever.
|
|
"""
|
|
out: list[str] = []
|
|
stack: list[object] = [document]
|
|
while stack:
|
|
node = stack.pop()
|
|
if isinstance(node, dict):
|
|
stack.extend(node.values())
|
|
elif isinstance(node, list):
|
|
stack.extend(node)
|
|
elif isinstance(node, str):
|
|
for match in _UNREACHABLE_REF.finditer(node):
|
|
out.append(
|
|
f"{path.relative_to(ROOT)}: citation names a path outside "
|
|
f"any repository: {match.group().strip(' ,;(')}"
|
|
)
|
|
return out
|
|
|
|
|
|
def _semantic_envelope_checks(path: Path, document: dict) -> list[str]:
|
|
if document.get("count") != len(document.get("items", [])):
|
|
return [f"{path.relative_to(ROOT)}: count does not equal len(items)"]
|
|
return []
|
|
|
|
|
|
def _semantic_database_checks(database: dict) -> list[str]:
|
|
out: list[str] = []
|
|
files = database.get("files", {})
|
|
if database.get("total_files") != len(files):
|
|
out.append("database.json: total_files does not equal len(files)")
|
|
if database.get("total_size") != sum(entry.get("size", 0) for entry in files.values()):
|
|
out.append("database.json: total_size does not equal the file-size sum")
|
|
# One path holds one content. Two entries naming the same path means one
|
|
# of them declares a hash the file at that path does not have.
|
|
owners: dict[str, str] = {}
|
|
for sha1, entry in files.items():
|
|
if entry.get("sha1") != sha1:
|
|
out.append(f"database.json: files/{sha1}: key and sha1 differ")
|
|
path = entry.get("path", "")
|
|
if not path:
|
|
continue
|
|
first = owners.setdefault(path, sha1)
|
|
if first != sha1:
|
|
out.append(
|
|
f"database.json: {path} is claimed by {first} and {sha1}"
|
|
)
|
|
return out
|
|
|
|
|
|
def _semantic_install_checks(path: Path, manifest: dict) -> list[str]:
|
|
out: list[str] = []
|
|
files = manifest.get("files", [])
|
|
if manifest.get("total_files") != len(files):
|
|
out.append(f"{path.relative_to(ROOT)}: total_files does not equal len(files)")
|
|
if manifest.get("total_size") != sum(entry.get("size", 0) for entry in files):
|
|
out.append(f"{path.relative_to(ROOT)}: total_size does not equal the file-size sum")
|
|
omitted = manifest.get("omitted_files", [])
|
|
if manifest.get("total_omitted") != len(omitted):
|
|
out.append(
|
|
f"{path.relative_to(ROOT)}: total_omitted does not equal "
|
|
"len(omitted_files)"
|
|
)
|
|
destinations = [entry.get("dest") for entry in files]
|
|
if len(destinations) != len(set(destinations)):
|
|
out.append(f"{path.relative_to(ROOT)}: duplicate destinations")
|
|
omitted_destinations = [entry.get("dest") for entry in omitted]
|
|
if len(omitted_destinations) != len(set(omitted_destinations)):
|
|
out.append(f"{path.relative_to(ROOT)}: duplicate omitted destinations")
|
|
overlap = set(destinations) & set(omitted_destinations)
|
|
if overlap:
|
|
out.append(
|
|
f"{path.relative_to(ROOT)}: destinations are both downloadable and omitted"
|
|
)
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--source-only",
|
|
action="store_true",
|
|
help="validate emulator/platform YAML only",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
errors: list[str] = []
|
|
errors.extend(
|
|
_validate_yaml_directory(ROOT / "emulators", "emulator.schema.json")
|
|
)
|
|
errors.extend(
|
|
_validate_yaml_directory(
|
|
ROOT / "platforms", "platform.schema.json", skip_private=True
|
|
)
|
|
)
|
|
|
|
if not args.source_only:
|
|
database_path = ROOT / "database.json"
|
|
errors.extend(_validate_json_files([database_path], "database.schema.json"))
|
|
if database_path.exists():
|
|
errors.extend(_semantic_database_checks(_load_json(database_path)))
|
|
|
|
install_paths = sorted((ROOT / "install").glob("*.json"))
|
|
errors.extend(
|
|
_validate_json_files(install_paths, "install-manifest.schema.json")
|
|
)
|
|
for path in install_paths:
|
|
errors.extend(_semantic_install_checks(path, _load_json(path)))
|
|
|
|
target_paths = sorted((ROOT / "install" / "targets").glob("*.json"))
|
|
errors.extend(
|
|
_validate_json_files(target_paths, "target-manifest.schema.json")
|
|
)
|
|
|
|
stats_path = ROOT / "docs" / "stats.json"
|
|
if stats_path.exists():
|
|
errors.extend(_validate_json_files([stats_path], "stats.schema.json"))
|
|
|
|
api_dir = ROOT / "docs" / "api" / "v1"
|
|
api_database = api_dir / "database.json"
|
|
if api_database.exists():
|
|
errors.extend(
|
|
_validate_json_files([api_database], "database.schema.json")
|
|
)
|
|
api_stats = api_dir / "stats.json"
|
|
if api_stats.exists():
|
|
errors.extend(_validate_json_files([api_stats], "stats.schema.json"))
|
|
envelope_paths = [
|
|
api_dir / name
|
|
for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json")
|
|
if (api_dir / name).exists()
|
|
]
|
|
errors.extend(
|
|
_validate_json_files(envelope_paths, "site-api-envelope.schema.json")
|
|
)
|
|
for path in envelope_paths:
|
|
errors.extend(_semantic_envelope_checks(path, _load_json(path)))
|
|
|
|
errors.extend(_validate_pack_manifests(ROOT / "dist"))
|
|
|
|
if errors:
|
|
for error in errors:
|
|
print(f"ERROR {error}")
|
|
print(f"{len(errors)} schema or semantic error(s)")
|
|
return 1
|
|
print("All RetroBIOS schemas and semantic invariants are valid.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|