diff --git a/scripts/generate_site.py b/scripts/generate_site.py index fca79e62..1a7324a5 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1413,6 +1413,47 @@ def dataset_structured_data(exports: list[dict], db: dict) -> dict: return data +def emulator_file_anchors(files: list[dict]) -> list[str]: + """The id of each file card on an emulator page, in profile order. + + A path declared once keeps its plain anchor. One declared for several + versions (raze's nine DUKE3D.GRP) takes what tells the version apart: + its hash, else its size, else, when even the size is shared, its + description. The page and its JSON-LD both read this, so a structured + part links to its card. + """ + bases = [ + "file-" + ( + re.sub( + r"[^a-z0-9]+", "-", + str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), + ).strip("-") or "entry" + ) + for f in files + ] + repeated = {b for b in bases if bases.count(b) > 1} + + def _version_key(f: dict) -> str: + digest = _first_hash_value(f) + if digest: + return digest[:8] + return f"{f['size']}b" if f.get("size") else "" + + keys = [(b, _version_key(f)) if b in repeated else (b, "") for f, b in zip(files, bases)] + used: dict[str, int] = {} + anchors = [] + for f, (base, key) in zip(files, keys): + anchor = f"{base}-{key}" if key else base + if base in repeated and keys.count((base, key)) > 1 and f.get("description"): + described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") + anchor = f"{anchor}-{described[:40].strip('-')}" + used[anchor] = used.get(anchor, 0) + 1 + if used[anchor] > 1: + anchor = f"{anchor}-{used[anchor]}" + anchors.append(anchor) + return anchors + + def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. @@ -1438,18 +1479,43 @@ def emulator_structured_data(profile: dict) -> dict: if profile.get("profiled_date"): data["dateModified"] = str(profile["profiled_date"]) parts = [] - for f in profile.get("files") or []: - if not isinstance(f, dict) or not f.get("name"): + files = [f for f in profile.get("files") or [] if isinstance(f, dict)] + for f, anchor in zip(files, emulator_file_anchors(files)): + if not f.get("name"): continue - identifiers = [] - for field in ("sha1", "sha256", "md5", "crc32"): - values = f.get(field) - for value in values if isinstance(values, list) else [values]: - if value: - identifiers.append(f"{field}:{str(value).lower()}") - part = {"@type": "CreativeWork", "name": str(f.get("path") or f["name"])} + # One identifier per accepted value: "sha256:a,b,c,d" matched no + # exact search (mesence's BS-X.bin). + identifiers = [ + f"{field}:{value}" + for field in ("sha1", "sha256", "md5", "crc32") + for value in _hash_values(f.get(field)) + ] + part = { + "@type": "CreativeWork", + "name": str(f.get("path") or f["name"]), + # The card's own anchor, as system and platform parts carry. + "url": f"#{anchor}", + } if identifiers: part["identifier"] = identifiers + # An archive is identified by its members: MAME and FBNeo hash them, + # never the ZIP, and 1 338 archive parts had no identifier at all. + members = [ + { + "@type": "CreativeWork", + "name": str(member.get("name", "")), + "identifier": [ + f"{field}:{value}" + for field in ("sha1", "crc32") + for value in _hash_values(member.get(field)) + ], + } + for member in f.get("contents") or [] + if isinstance(member, dict) and member.get("name") + and (member.get("sha1") or member.get("crc32")) + ] + if members: + part["hasPart"] = members parts.append(part) if parts: data["hasPart"] = parts @@ -3016,38 +3082,7 @@ def generate_emulator_page( lines.extend(_emulator_file_summary(files, _file_available)) # File table - bases = [ - "file-" + ( - re.sub( - r"[^a-z0-9]+", "-", - str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), - ).strip("-") or "entry" - ) - for f in files - ] - repeated = {b for b in bases if bases.count(b) > 1} - - # A path declared once keeps its plain anchor. One declared for - # several versions (raze's nine DUKE3D.GRP) takes what tells the - # version apart: its hash, else its size, else, when even the size - # is shared, its description. By rank, a link to -3 named another - # edition as soon as a version was declared before it. - def _version_key(f: dict) -> str: - digest = _first_hash_value(f) - if digest: - return digest[:8] - return f"{f['size']}b" if f.get("size") else "" - - keys = [(b, _version_key(f)) if b in repeated else (b, "") for f, b in zip(files, bases)] - used: dict[str, int] = {} - for f, (base, key) in zip(files, keys): - anchor = f"{base}-{key}" if key else base - if base in repeated and keys.count((base, key)) > 1 and f.get("description"): - described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") - anchor = f"{anchor}-{described[:40].strip('-')}" - used[anchor] = used.get(anchor, 0) + 1 - if used[anchor] > 1: - anchor = f"{anchor}-{used[anchor]}" + for f, anchor in zip(files, emulator_file_anchors(files)): lines.extend( _render_emulator_file( f, profile, consumers, files, _file_available, anchor, name diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index abf8658c..340c8723 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -532,5 +532,29 @@ class AVersionPerBuildReadsAsVersions(unittest.TestCase): self.assertNotIn("{'standalone'", page) +class EmulatorPartsCarryTheirIdentity(unittest.TestCase): + """The emulator JSON-LD published 'sha256:a,b,c,d' as one identifier, + gave archives none though their members carry crc32 and sha1, and gave + no part the url of its card.""" + + def test_identifiers_members_and_url(self): + from generate_site import emulator_file_anchors, emulator_structured_data # noqa: PLC0415 + + files = [ + {"name": "BS-X.bin", "sha256": "aa,bb"}, + {"name": "neogeo.zip", "contents": [ + {"name": "sp-s2.sp1", "crc32": "9036D879", "sha1": "ff" * 20}]}, + ] + data = emulator_structured_data({"files": files}) + bsx, neogeo = data["hasPart"] + self.assertEqual(bsx["identifier"], ["sha256:aa", "sha256:bb"]) + self.assertEqual( + neogeo["hasPart"][0]["identifier"], ["sha1:" + "ff" * 20, "crc32:9036d879"] + ) + self.assertEqual( + [bsx["url"], neogeo["url"]], [f"#{a}" for a in emulator_file_anchors(files)] + ) + + if __name__ == "__main__": unittest.main()