#!/usr/bin/env python3
"""Native-format exporters.

export_native.py rewrites each platform's own file, corrected: Recalbox's
es_bios.xml, Batocera's systems list, RetroDECK's manifests. The point is
that a maintainer can take the output and use it, so what matters is
whether the platform would still read it, and whether anything it already
declared went missing on the way through.

The fidelity cases read the platforms' real files from the upstream cache
and skip when it has not been populated (python scripts/export_native.py
--all --fetch), because a test that needs the network is not a test.
"""

from __future__ import annotations

import json
import os
import re
import subprocess
import sys
import tempfile
import unittest
import xml.etree.ElementTree as ET
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))

from common import (  # noqa: E402
    list_registered_platforms,
    load_platform_config,
    parse_untrusted_xml,
)
from exporter import discover_exporters  # noqa: E402
from exporter.base_exporter import BaseExporter  # noqa: E402
from export_native import (  # noqa: E402
    SOURCES_INDEX,
    collect_originals,
    fetch,
    pinned_base,
    source_key,
)
from exporter.baseline import (  # noqa: E402
    NativeFile,
    build_native_model,
)

UPSTREAM_CACHE = REPO_ROOT / ".cache" / "upstream-native"

TRUTH = {
    "systems": {
        "sony-playstation": {
            "files": [
                {
                    "name": "scph5501.bin",
                    "path": "psx/scph5501.bin",
                    "md5": "490f666e1afb15b7362b406ed1cea246",
                    "size": 524288,
                    "required": True,
                    "_cores": ["duckstation"],
                },
                {
                    "name": "scph7001.bin",
                    "path": "psx/scph7001.bin",
                    "md5": "1e68c231d0896b7eadcad1d7d8e76129",
                    "size": 524288,
                    "required": False,
                    "_cores": ["duckstation"],
                },
                # Carries an md5 on purpose: without one it would be dropped
                # for being incomplete, and the placeholder guard would never
                # be the reason it stayed out.
                {
                    "name": "<user-selected>.bin",
                    "path": "psx/PLACEHOLDER-MUST-NOT-SHIP.bin",
                    "md5": "00000000000000000000000000000000",
                    "size": 1,
                    "required": False,
                    "_cores": ["duckstation"],
                },
            ],
        },
    },
}

SCRAPED = {
    "systems": {
        "sony-playstation": {
            "native_id": "psx",
            "files": [
                {
                    "name": "scph5501.bin",
                    "destination": "psx/scph5501.bin",
                    "native_system": "psx",
                    "native_name": "Sony PlayStation",
                    "md5": "deadbeefdeadbeefdeadbeefdeadbeef",
                    "core": "libretro/duckstation",
                    "required": True,
                },
                {
                    "name": "unknown-to-truth.bin",
                    "destination": "psx/unknown-to-truth.bin",
                    "native_system": "psx",
                    "native_name": "Sony PlayStation",
                    "md5": "11111111111111111111111111111111",
                    "core": "libretro/duckstation",
                    "required": True,
                },
            ],
        },
    },
}


def model(truth: dict = TRUTH, scraped: dict | None = SCRAPED):
    return build_native_model(truth, scraped)


class Reconciliation(unittest.TestCase):
    """The platform's file corrected, not replaced by ours."""

    def test_a_bare_truth_name_is_not_a_path_suffix(self):
        """fbneo's boot.bin has no path; libretro's dc/boot.bin ends in
        /boot.bin. The suffix paired them past the size guard and wrote the
        480-byte file's size and hashes over the 2 MB Dreamcast BIOS."""
        truth = {"systems": {"sega-dreamcast": {"files": [
            {"name": "boot.bin", "size": 480, "crc32": "f0774fc2"}]}}}
        scraped = {"systems": {"sega-dreamcast": {"native_id": "dc", "files": [
            {"name": "boot.bin", "destination": "dc/boot.bin", "native_system": "dc",
             "size": 2097152, "crc32": "89f2b1a1"}]}}}
        systems, report = build_native_model(truth, scraped)
        platform_entry = next(
            fe for fe in systems["dc"].files if fe.destination == "dc/boot.bin"
        )
        self.assertIsNone(platform_entry.truth)
        self.assertEqual(report.hashes_corrected, [])

    def test_a_file_the_truth_says_nothing_about_is_kept(self):
        systems, _ = model()
        names = {fe.name for fe in systems["psx"].files}
        self.assertIn("unknown-to-truth.bin", names)

    def test_the_truth_overrides_a_hash_the_platform_got_wrong(self):
        systems, report = model()
        entry = next(fe for fe in systems["psx"].files if fe.name == "scph5501.bin")
        self.assertEqual(entry.hash("md5"), "490f666e1afb15b7362b406ed1cea246")
        self.assertTrue(any("scph5501" in c for c in report.hashes_corrected))

    def test_a_file_only_the_truth_knows_is_added(self):
        systems, report = model()
        names = {fe.name for fe in systems["psx"].files}
        self.assertIn("scph7001.bin", names)
        self.assertEqual(report.files_added, 1)

    def test_a_placeholder_is_never_added(self):
        systems, _ = model()
        names = {fe.name for fe in systems["psx"].files}
        self.assertNotIn("<user-selected>.bin", names)

    def test_systems_are_keyed_by_the_name_the_platform_uses(self):
        systems, _ = model()
        self.assertEqual(sorted(systems), ["psx"])

    def test_the_platform_own_grouping_survives_a_collapsed_slug(self):
        """Several native systems can share one of our slugs.

        Recalbox files pcengine, pcenginecd and supergrafx under one slug.
        Grouping on the slug would publish one of the three and drop the
        rest, which is how eleven Recalbox platforms went missing.
        """
        scraped = {
            "systems": {
                "nec-pc-engine": {
                    "native_id": "pcengine",
                    "files": [
                        {
                            "name": "syscard3.pce",
                            "destination": "pcengine/syscard3.pce",
                            "native_system": "pcengine",
                            "md5": "38179df8f4ac870017db21ebcbf53114",
                        },
                        {
                            "name": "syscard3.pce",
                            "destination": "supergrafx/syscard3.pce",
                            "native_system": "supergrafx",
                            "md5": "38179df8f4ac870017db21ebcbf53114",
                        },
                    ],
                },
            },
        }
        systems, _ = build_native_model({"systems": {}}, scraped)
        self.assertEqual(sorted(systems), ["pcengine", "supergrafx"])

    def test_an_entry_the_platform_declares_without_a_hash_is_still_written(self):
        """Batocera and RetroBat both ship entries with an empty md5."""
        scraped = {
            "systems": {
                "nintendo-ds": {
                    "native_id": "nds",
                    "files": [
                        {
                            "name": "firmware.bin",
                            "destination": "firmware.bin",
                            "native_system": "nds",
                        }
                    ],
                }
            }
        }
        systems, _ = build_native_model({"systems": {}}, scraped)
        entry = systems["nds"].files[0]
        self.assertTrue(BaseExporter.writable(entry, require="md5"))

    def test_no_format_drops_what_the_platform_already_declares(self):
        """An export poorer than the file it corrects is not usable."""
        systems, _ = model()
        for name, cls in discover_exporters().items():
            for system in systems.values():
                for entry in system.files:
                    if entry.platform is None:
                        continue
                    with self.subTest(platform=name, file=entry.name):
                        self.assertTrue(cls().writable(entry))

    def test_an_addition_the_format_cannot_express_is_refused(self):
        truth = {
            "systems": {
                "nintendo-ds": {"files": [{"name": "no-hash.bin", "required": True}]}
            }
        }
        systems, _ = build_native_model(truth, None)
        entry = systems["nintendo-ds"].files[0]
        self.assertFalse(BaseExporter.writable(entry, require="md5"))


class ExporterRegistry(unittest.TestCase):
    """Every registered platform has an exporter, and it declares itself."""

    def test_every_exporter_declares_a_platform_and_a_filename(self):
        for name, cls in discover_exporters().items():
            with self.subTest(platform=name):
                self.assertTrue(cls.platform_name())
                self.assertTrue(cls.native_filename())
                self.assertTrue(issubclass(cls, BaseExporter))

    def test_every_registered_platform_can_be_exported(self):
        """A platform with no exporter is a platform we cannot hand back.

        Archived included: the export is a correction we offer, not a
        release artefact, and RetroPie's users are not fewer for it.
        """
        exporters = discover_exporters()
        registered = set(
            list_registered_platforms(
                str(REPO_ROOT / "platforms"), include_archived=True
            )
        )
        self.assertEqual(registered - set(exporters), set())

    def test_no_exporter_is_registered_for_an_unregistered_platform(self):
        """A dead exporter is one nobody will notice has stopped working."""
        exporters = set(discover_exporters())
        registered = set(
            list_registered_platforms(
                str(REPO_ROOT / "platforms"), include_archived=True
            )
        )
        self.assertEqual(exporters - registered, set())

    def test_every_exporter_says_what_its_format_can_state(self):
        """A correction to a field the file has no place for is not delivered."""
        for name, cls in discover_exporters().items():
            with self.subTest(platform=name):
                carried = cls.carries()
                self.assertTrue(carried)
                self.assertTrue(
                    carried
                    <= {"size", "crc32", "md5", "sha1", "sha256", "required", "name"},
                    f"{name} claims a field that is not one of ours: {carried}",
                )

    def test_only_a_format_with_the_field_claims_to_carry_it(self):
        """Recalbox writes mandatory; RetroBat's entries are md5 and file."""
        exporters = discover_exporters()
        self.assertIn("required", exporters["recalbox"].carries())
        self.assertNotIn("required", exporters["retrobat"].carries())
        self.assertNotIn("required", exporters["retroarch"].carries())

    def test_an_exporter_that_patches_code_says_it_needs_the_original(self):
        """Regenerating a script from data alone drops the code around it."""
        for name in ("batocera", "rocknix", "emudeck", "retrodeck", "bizhawk",
                     "misterfpga"):
            with self.subTest(platform=name):
                self.assertTrue(discover_exporters()[name].needs_original())


class SearchOrder(unittest.TestCase):
    """A list is read in the order the code looks, best first."""

    @staticmethod
    def _entry(name, priority=None, position=0):
        truth = {"name": name, "required": True, "md5": f"{position:032x}"}
        if priority is not None:
            truth["priority"] = priority
        return NativeFile(
            name=name, destination=name, native_system="psx", truth=truth
        )

    def test_the_declared_rank_orders_the_names(self):
        exporter = discover_exporters()["retropie"]()
        candidates = [
            self._entry("scph5502.bin", 5, 0),
            self._entry("scph5501.bin", 2, 1),
            self._entry("scph7001.bin", 10, 2),
        ]
        self.assertEqual(
            exporter._in_search_order(candidates),
            ["scph5501.bin", "scph5502.bin", "scph7001.bin"],
        )

    def test_without_a_rank_the_order_the_code_walks_is_kept(self):
        """An undeclared order is the profile's own, not the alphabet."""
        exporter = discover_exporters()["retropie"]()
        candidates = [
            self._entry("zzz.bin", None, 0),
            self._entry("aaa.bin", None, 1),
        ]
        self.assertEqual(
            exporter._in_search_order(candidates), ["zzz.bin", "aaa.bin"]
        )

    def test_a_ranked_file_is_read_before_an_unranked_one(self):
        exporter = discover_exporters()["retropie"]()
        candidates = [
            self._entry("unranked.bin", None, 0),
            self._entry("ranked.bin", 9, 1),
        ]
        self.assertEqual(
            exporter._in_search_order(candidates), ["ranked.bin", "unranked.bin"]
        )

    def test_cores_that_disagree_keep_the_best_rank(self):
        """Nothing is dropped on the rank here, so the best one wins.

        slot.py voids the choice when cores disagree because it keeps one
        file; ordering keeps them all, and a file that is some core's first
        choice belongs early.
        """
        merged = {"name": "scph5501.bin", "priority": 2, "priority_conflict": True}
        entry = NativeFile(
            name="scph5501.bin",
            destination="scph5501.bin",
            native_system="psx",
            truth=merged,
        )
        self.assertEqual(entry.priority, 2)


class AdditionOrder(unittest.TestCase):
    """What we add is listed the way the code looks for it."""

    TRUTH = {
        "systems": {
            "sony-playstation": {
                "files": [
                    {
                        "name": "third.bin",
                        "md5": "c" * 32,
                        "required": True,
                        "priority": 10,
                    },
                    {
                        "name": "first.bin",
                        "md5": "a" * 32,
                        "required": True,
                        "priority": 1,
                    },
                    {"name": "unranked.bin", "md5": "b" * 32, "required": True},
                ]
            }
        }
    }

    def test_a_ranked_addition_is_named_before_a_later_one(self):
        systems, _ = build_native_model(self.TRUTH, None)
        names = [fe.name for fe in systems["sony-playstation"].files]
        self.assertEqual(names, ["first.bin", "third.bin", "unranked.bin"])

    def test_what_the_platform_wrote_keeps_its_place(self):
        """Reordering their own entries would be churn, not a correction."""
        scraped = {
            "systems": {
                "sony-playstation": {
                    "native_id": "psx",
                    "files": [
                        {
                            "name": "theirs-b.bin",
                            "destination": "theirs-b.bin",
                            "native_system": "psx",
                            "md5": "d" * 32,
                        },
                        {
                            "name": "theirs-a.bin",
                            "destination": "theirs-a.bin",
                            "native_system": "psx",
                            "md5": "e" * 32,
                        },
                    ],
                }
            }
        }
        systems, _ = build_native_model(self.TRUTH, scraped)
        names = [fe.name for fe in systems["psx"].files]
        self.assertEqual(names[:2], ["theirs-b.bin", "theirs-a.bin"])
        self.assertEqual(names[2], "first.bin")


class PinnedRevision(unittest.TestCase):
    """The file we patch is the revision our transcription came from."""

    def test_the_tag_the_scraper_pinned_wins_over_the_branch_tip(self):
        wanted = {"batocera-systems": "https://example.invalid/master/batocera-systems"}
        scraped = {
            "source": "https://raw.githubusercontent.com/batocera-linux"
            "/batocera.linux/batocera-43.1/package/batocera-systems"
        }
        self.assertEqual(
            pinned_base(wanted, scraped),
            "https://raw.githubusercontent.com/batocera-linux/batocera.linux"
            "/batocera-43.1/package/",
        )

    def test_a_blob_page_is_read_as_raw(self):
        """github.com/.../blob/... names a revision but serves HTML."""
        wanted = {"System.dat": "https://example.invalid/System.dat"}
        scraped = {
            "source": "https://github.com/libretro/libretro-database"
            "/blob/master/dat/System.dat"
        }
        self.assertEqual(
            pinned_base(wanted, scraped),
            "https://raw.githubusercontent.com/libretro/libretro-database"
            "/master/dat/",
        )

    def test_a_source_that_names_no_wanted_file_changes_nothing(self):
        wanted = {"component_manifest.json": "https://example.invalid/x.json"}
        scraped = {"source": "https://github.com/RetroDECK/components"}
        self.assertEqual(pinned_base(wanted, scraped), "")

    def test_every_platform_that_records_its_source_pins_the_export(self):
        """A scraped platform states the revision; the export follows it."""
        exporters = discover_exporters()
        for name in ("batocera", "recalbox", "bizhawk", "romm", "retroarch"):
            with self.subTest(platform=name):
                config = load_platform_config(name, str(REPO_ROOT / "platforms"))
                base = pinned_base(dict(exporters[name].native_sources()), config)
                self.assertTrue(base.startswith("https://"), base)


class _OneFileExporter:
    """An exporter reduced to what collect_originals reads."""

    def __init__(self, url: str):
        self._url = url

    def platform_name(self) -> str:
        return "plat"

    def native_sources(self) -> dict[str, str]:
        return {"list.txt": self._url}


class CachedOriginal(unittest.TestCase):
    """The cached original is the file the platform data was read from."""

    def setUp(self):
        self._tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self._tmp.cleanup)
        self.root = Path(self._tmp.name)
        self.cache = self.root / "cache"
        self.index = self.cache / SOURCES_INDEX

    def _upstream(self, name: str, text: str) -> str:
        path = self.root / name
        path.write_text(text, encoding="utf-8")
        return path.as_uri()

    def test_the_same_url_is_served_from_the_cache(self):
        url = self._upstream("v1.txt", "one")
        target = self.cache / "plat" / "list.txt"
        self.assertEqual(fetch(url, target, self.index), b"one")
        self._upstream("v1.txt", "moved")
        self.assertEqual(fetch(url, target, self.index), b"one")

    def test_refresh_reads_a_branch_url_again(self):
        """A branch URL keeps its name while the platform moves."""
        url = self._upstream("branch.txt", "one")
        target = self.cache / "plat" / "list.txt"
        fetch(url, target, self.index)
        self._upstream("branch.txt", "two")
        self.assertEqual(fetch(url, target, self.index, refresh=True), b"two")
        self.assertEqual(target.read_bytes(), b"two")

    def test_a_new_pin_refetches_an_existing_file(self):
        """Reading the file straight from disk served the old pin forever."""
        old = self._upstream("tag-1.txt", "one")
        new = self._upstream("tag-2.txt", "two")
        originals, missing = collect_originals(
            _OneFileExporter(old), {}, self.cache, True
        )
        self.assertEqual((originals, missing), ({"list.txt": "one"}, []))
        originals, missing = collect_originals(
            _OneFileExporter(new), {}, self.cache, True
        )
        self.assertEqual((originals, missing), ({"list.txt": "two"}, []))

    def test_offline_serves_what_is_cached_and_names_what_is_not(self):
        url = self._upstream("v1.txt", "one")
        _, missing = collect_originals(_OneFileExporter(url), {}, self.cache, False)
        self.assertEqual(missing, ["list.txt: absent and fetching is off"])
        collect_originals(_OneFileExporter(url), {}, self.cache, True)
        originals, missing = collect_originals(
            _OneFileExporter("file:///nowhere/else.txt"), {}, self.cache, False
        )
        self.assertEqual((originals, missing), ({"list.txt": "one"}, []))

    def test_the_index_names_a_file_the_same_from_any_directory(self):
        target = self.cache / "plat" / "list.txt"
        self.assertEqual(source_key(target, self.index), "plat/list.txt")
        relative = Path(os.path.relpath(target, Path.cwd()))
        self.assertEqual(source_key(relative, self.index), "plat/list.txt")


class RecalboxExport(unittest.TestCase):
    def _render(self):
        systems, report = model()
        exporter = discover_exporters()["recalbox"]()
        return exporter, systems, exporter.render(systems, report, {}, None)

    def test_output_is_well_formed_xml_in_recalbox_shape(self):
        _, _, produced = self._render()
        root = ET.fromstring(produced["es_bios.xml"])
        self.assertEqual(root.tag, "biosList")
        self.assertTrue(list(root.iter("system")))

    def test_every_element_carries_what_the_schema_requires(self):
        """es_bios.xsd makes path, md5 and core required on every bios."""
        _, _, produced = self._render()
        root = ET.fromstring(produced["es_bios.xml"])
        for element in root.iter("bios"):
            for attribute in ("path", "md5", "core"):
                self.assertTrue(element.get(attribute), attribute)

    def test_mandatory_is_only_stated_when_it_is_false(self):
        _, _, produced = self._render()
        root = ET.fromstring(produced["es_bios.xml"])
        by_path = {e.get("path"): e for e in root.iter("bios")}
        self.assertIsNone(by_path["psx/scph5501.bin"].get("mandatory"))
        self.assertEqual(by_path["psx/scph7001.bin"].get("mandatory"), "false")

    def test_hash_match_mandatory_does_not_follow_mandatory(self):
        """They are separate axes: Recalbox uses all four combinations."""
        _, _, produced = self._render()
        root = ET.fromstring(produced["es_bios.xml"])
        optional = next(
            e for e in root.iter("bios") if e.get("mandatory") == "false"
        )
        self.assertIsNone(optional.get("hashMatchMandatory"))

    def test_the_export_passes_its_own_validator(self):
        exporter, systems, produced = self._render()
        self.assertEqual(exporter.validate(systems, produced), [])

    def test_an_addition_goes_where_the_core_reads_it(self):
        """DuckStation reads scph5000.bin at the BIOS root; the export wrote
        psx/scph5000.bin, a path no core opens."""
        import copy  # noqa: PLC0415

        truth = copy.deepcopy(TRUTH)
        truth["systems"]["sony-playstation"]["files"].append({
            "name": "scph5000.bin", "md5": "57a06303dfa9cf9351222dfcbb4a29d9",
            "size": 524288, "required": False, "_cores": ["duckstation"],
        })
        systems, report = build_native_model(truth, SCRAPED)
        produced = discover_exporters()["recalbox"]().render(systems, report, {}, None)
        paths = {e.get("path") for e in ET.fromstring(produced["es_bios.xml"]).iter("bios")}
        self.assertIn("scph5000.bin", paths)
        self.assertNotIn("psx/scph5000.bin", paths)

    def test_two_blocks_of_one_platform_stay_two(self):
        """Recalbox files Pentagon and Scorpion under zxspectrum, apart.

        Merged, the Scorpion ROMs sat under the Pentagon name and the BIOS
        manager lost a section; the line the maintainer commented out went
        with the rewrite.
        """
        old, new = "1" * 32, "2" * 32

        def declared(name, label, md5):
            return {
                "name": name, "destination": f"fuse/{name}",
                "native_system": "zxspectrum", "native_name": label,
                "native_path": f"fuse/{name}", "md5": md5,
                "core": "libretro/fuse", "required": False,
            }

        pentagon = "Pentagon 128/512/1024 (ZX Spectrum clone)"
        scorpion = "Scorpion 256K (ZX Spectrum clone)"
        scraped = {"systems": {"sinclair-zxspectrum": {
            "native_id": "zxspectrum",
            "files": [
                declared("128p-0.rom", pentagon, "3" * 32),
                declared("256s-0.rom", scorpion, old),
            ],
        }}}
        truth = {"systems": {"sinclair-zxspectrum": {"files": [{
            "name": "256s-0.rom", "path": "fuse/256s-0.rom", "md5": new,
            "required": False, "_cores": ["fuse"],
        }]}}}
        original = (
            '<?xml version="1.0" encoding="UTF-8"?>\n<biosList>\n'
            f'  <system fullname="{pentagon}" platform="zxspectrum">\n'
            f'    <bios path="fuse/128p-0.rom" md5="{"3" * 32}" core="libretro/fuse" />\n'
            '    <!--bios path="fuse/gluck.rom" md5="x" core="libretro/fuse" /-->\n'
            "  </system>\n"
            f'  <system fullname="{scorpion}" platform="zxspectrum">\n'
            f'    <bios path="fuse/256s-0.rom" md5="{old}" core="libretro/fuse" />\n'
            "  </system>\n</biosList>\n"
        )
        systems, report = build_native_model(truth, scraped)
        produced = discover_exporters()["recalbox"]().render(
            systems, report, {"es_bios.xml": original}, None
        )["es_bios.xml"]
        self.assertIn('<!--bios path="fuse/gluck.rom"', produced)
        blocks = {
            s.get("fullname"): [b.get("md5") for b in s.iter("bios")]
            for s in ET.fromstring(produced).iter("system")
        }
        self.assertEqual(blocks, {pentagon: ["3" * 32], scorpion: [new]})

    def test_a_truncated_export_is_reported(self):
        exporter, systems, _ = self._render()
        empty = {
            "es_bios.xml": '<?xml version="1.0"?>\n<biosList></biosList>\n',
        }
        self.assertTrue(exporter.validate(systems, empty))


class RommExport(unittest.TestCase):
    def _render(self):
        systems, report = model()
        exporter = discover_exporters()["romm"]()
        return exporter, systems, exporter.render(systems, report, {}, None)

    def test_every_entry_can_actually_verify(self):
        """RomM matches on size and then on one hash; without both, never."""
        _, _, produced = self._render()
        data = json.loads(produced["known_bios_files.json"])
        self.assertTrue(data)
        for key, entry in data.items():
            with self.subTest(key=key):
                self.assertTrue(entry.get("size"))
                self.assertTrue(
                    any(entry.get(h) for h in ("md5", "sha1", "crc"))
                )

    def test_size_is_written_as_the_fixture_writes_it(self):
        _, _, produced = self._render()
        data = json.loads(produced["known_bios_files.json"])
        self.assertIsInstance(next(iter(data.values()))["size"], str)

    def test_keys_are_the_platform_slug_the_file_belongs_to(self):
        _, _, produced = self._render()
        data = json.loads(produced["known_bios_files.json"])
        self.assertTrue(all(k.startswith("psx:") for k in data))


class BizhawkExport(unittest.TestCase):
    SHA1 = "a" * 40

    def _export(self, name, call):
        scraped = {"systems": {"nintendo-ds": {"native_id": "NDS", "files": [{
            "name": name, "destination": name, "native_system": "NDS",
        }]}}}
        truth = {"systems": {"nintendo-ds": {"files": [{
            "name": name, "path": name, "sha1": self.SHA1,
            "required": True, "_cores": ["melonds"],
        }]}}}
        systems, report = build_native_model(truth, scraped)
        (entry,) = systems["NDS"].files
        self.assertIn("sha1", entry.filled)
        exporter = discover_exporters()["bizhawk"]()
        source = f"class FirmwareDatabase {{\n    {call}\n}}\n"
        produced = exporter.render(
            systems, report, {"FirmwareDatabase.cs": source}, None
        )["FirmwareDatabase.cs"]
        return exporter, entry, source, produced

    def test_a_fill_a_dummy_call_cannot_hold_is_not_counted(self):
        """BizHawk states DSi_Firmware.bin with SHA1Checksum.Dummy."""
        exporter, entry, source, produced = self._export(
            "DSi_Firmware.bin",
            'FirmwareAndOption(SHA1Checksum.Dummy, 131072, "NDS", "firmwarei", '
            '"DSi_Firmware.bin", "DSi Firmware");',
        )
        self.assertEqual(produced, source)
        self.assertFalse(exporter.states(entry, "sha1"))

    def test_a_fill_the_call_holds_is_counted(self):
        exporter, entry, _, produced = self._export(
            "bios.bin", f'File("{"0" * 40}", 4096, "bios.bin", "BIOS");'
        )
        self.assertIn(self.SHA1.upper(), produced)
        self.assertTrue(exporter.states(entry, "sha1"))


class RetrodeckExport(unittest.TestCase):
    A, B, C = "a" * 32, "b" * 32, "c" * 32

    def _render(self, declared, accepted):
        """Export one file RetroDECK lists as an md5 array."""
        truth = {
            "systems": {
                "atari-400-800": {
                    "files": [
                        {
                            "name": "ATARIOSB.ROM",
                            "path": "ATARIOSB.ROM",
                            "md5": accepted,
                            "required": True,
                            "_cores": ["atari800"],
                        }
                    ]
                }
            }
        }
        scraped = {
            "systems": {
                "atari-400-800": {
                    "native_id": "atari800",
                    "files": [
                        {
                            "name": "ATARIOSB.ROM",
                            "destination": "bios/ATARIOSB.ROM",
                            "native_system": "atari800",
                            "md5": ",".join(declared),
                            "component": "retroarch",
                            "required": True,
                        }
                    ],
                }
            }
        }
        original = json.dumps({
            "retroarch": {
                "bios": [
                    {
                        "filename": "ATARIOSB.ROM",
                        "md5": declared,
                        "system": "atari800",
                        "required": "Required",
                    }
                ]
            }
        })
        systems, report = build_native_model(truth, scraped)
        exporter = discover_exporters()["retrodeck"]()
        produced = exporter.render(
            systems, report, {"retroarch/component_manifest.json": original}, None
        )
        (entry,) = json.loads(produced["retroarch/component_manifest.json"])[
            "retroarch"
        ]["bios"]
        return entry["md5"]

    def test_an_unchanged_md5_array_is_left_as_written(self):
        """The same dumps in another order are no correction."""
        self.assertEqual(
            self._render([self.A, self.B], [self.B, self.A]), [self.A, self.B]
        )

    def test_a_corrected_md5_array_stays_an_array(self):
        """A comma-joined string reads back as one value that is no md5."""
        written = self._render([self.A, self.B], [self.A, self.C])
        self.assertIsInstance(written, list)
        self.assertEqual(set(written), {self.A, self.B, self.C})


class BatoceraExport(unittest.TestCase):
    def _render(self, originals):
        systems, report = model()
        exporter = discover_exporters()["batocera"]()
        return exporter, systems, exporter.render(systems, report, originals, None)

    SKELETON = (
        "#!/usr/bin/env python\n"
        "systems = {\n"
        "}\n"
        "def checkBios(systems, prefix, filterROMs):\n"
        "    return {}\n"
    )

    def test_an_unhashed_entry_is_written_with_an_empty_md5(self):
        scraped = {
            "systems": {
                "nintendo-ds": {
                    "native_id": "nds",
                    "files": [
                        {
                            "name": "firmware.bin",
                            "destination": "firmware.bin",
                            "native_system": "nds",
                        }
                    ],
                }
            }
        }
        systems, report = build_native_model({"systems": {}}, scraped)
        exporter = discover_exporters()["batocera"]()
        produced = exporter.render(
            systems, report, {"batocera-systems": self.SKELETON}, None
        )
        self.assertIn('"md5": ""', produced["batocera-systems"])

    def test_a_blank_md5_is_not_filled_with_one_of_several_dumps(self):
        """checkBios compares one md5: the first of two refuses the other.

        The empty field checks by existence, so a user holding the second
        dump the core accepts would turn from OK to UNTESTED.
        """
        first, second = "1" * 32, "2" * 32
        truth = {
            "systems": {
                "nintendo-ds": {
                    "files": [
                        {
                            "name": "firmware.bin",
                            "path": "firmware.bin",
                            "md5": [first, second],
                            "required": True,
                            "_cores": ["melonds"],
                        }
                    ]
                }
            }
        }
        scraped = {
            "systems": {
                "nintendo-ds": {
                    "native_id": "nds",
                    "files": [
                        {
                            "name": "firmware.bin",
                            "destination": "firmware.bin",
                            "native_system": "nds",
                            "md5": "",
                        }
                    ],
                }
            }
        }
        for platform, filename in (
            ("batocera", "batocera-systems"),
            ("rocknix", "rocknix-systems"),
        ):
            with self.subTest(platform=platform):
                systems, report = build_native_model(truth, scraped)
                (entry,) = systems["nds"].files
                self.assertIn("md5", entry.filled)
                exporter = discover_exporters()[platform]()
                produced = exporter.render(
                    systems, report, {filename: self.SKELETON}, None
                )
                self.assertIn('"md5": ""', produced[filename])
                self.assertNotIn(first, produced[filename])
                self.assertFalse(exporter.states(entry, "md5"))

    def test_a_system_the_platform_does_not_name_gets_no_entry(self):
        """checkBios walks the systems es_systems declares, not our slugs.

        An addition filed under nintendo-snes beside Batocera's own snes is
        never checked, so it is counted as one the format cannot state.
        """
        truth = {
            "systems": {
                **TRUTH["systems"],
                "nintendo-snes": {
                    "files": [
                        {
                            "name": "st010.zip",
                            "path": "st010.zip",
                            "md5": "2" * 32,
                            "required": False,
                            "_cores": ["snes9x"],
                        }
                    ]
                },
            }
        }
        systems, report = build_native_model(truth, SCRAPED)
        (foreign,) = systems["nintendo-snes"].files
        self.assertTrue(foreign.foreign_system)
        own = next(fe for fe in systems["psx"].files if fe.name == "scph7001.bin")
        self.assertFalse(own.foreign_system)
        for platform in ("batocera", "rocknix", "retrobat", "recalbox"):
            with self.subTest(platform=platform):
                exporter = discover_exporters()[platform]
                self.assertFalse(exporter().writable(foreign))
                self.assertTrue(exporter().writable(own))
        # A format that finds files by path has no system key to miss.
        self.assertTrue(discover_exporters()["retroarch"]().writable(foreign))
        produced = discover_exporters()["batocera"]().render(
            systems, report, {"batocera-systems": self.SKELETON}, None
        )
        self.assertNotIn('"nintendo-snes"', produced["batocera-systems"])

    def test_a_system_the_platform_names_without_files_is_its_own(self):
        """A system the platform declares empty still has the platform's id."""
        scraped = {"systems": {"nintendo-snes": {"native_id": "snes", "files": []}}}
        truth = {"systems": {"nintendo-snes": {"files": [{
            "name": "st010.zip", "path": "st010.zip", "md5": "2" * 32,
            "required": False, "_cores": ["snes9x"],
        }]}}}
        systems, _ = build_native_model(truth, scraped)
        (entry,) = systems["snes"].files
        self.assertFalse(entry.foreign_system)
        self.assertTrue(discover_exporters()["batocera"]().writable(entry))

    def test_writing_without_the_platform_file_is_refused(self):
        """A systems dict alone is not the script Batocera runs."""
        systems, report = model()
        exporter = discover_exporters()["batocera"]()
        with self.assertRaises(ValueError):
            exporter.render(systems, report, {}, None)

    def test_the_code_around_the_data_survives(self):
        original = (
            "#!/usr/bin/env python\n"
            "systems = {\n"
            '    "psx": { "name": "PS1", "biosFiles": [ '
            '{ "md5": "deadbeefdeadbeefdeadbeefdeadbeef", '
            '"file": "bios/psx/scph5501.bin" } ] },\n'
            "}\n"
            "def checkBios(systems, prefix, filterROMs):\n"
            "    return {}\n"
        )
        _, _, produced = self._render({"batocera-systems": original})
        output = produced["batocera-systems"]
        self.assertIn("def checkBios(", output)
        self.assertIn("#!/usr/bin/env python", output)

    def test_a_note_beside_a_file_is_not_lost_when_the_entry_changes(self):
        original = (
            "systems = {\n"
            '    "psx": { "name": "PS1", "biosFiles": [\n'
            '        # ideally the other revision\n'
            '        { "md5": "deadbeefdeadbeefdeadbeefdeadbeef", '
            '"file": "bios/psx/scph5501.bin" } ] },\n'
            "}\n"
            "def checkBios(systems, prefix, filterROMs):\n"
            "    return {}\n"
        )
        _, _, produced = self._render({"batocera-systems": original})
        self.assertIn("# ideally the other revision", produced["batocera-systems"])


class UpstreamFidelity(unittest.TestCase):
    """The real files, compared against what we hand back.

    Nothing a platform declares today may be missing from the export: an
    export that loses systems is one no maintainer can apply.
    """

    @classmethod
    def setUpClass(cls):
        if not UPSTREAM_CACHE.is_dir():
            raise unittest.SkipTest(
                "no upstream cache; run scripts/export_native.py --all --fetch"
            )

    def _export(self, platform: str, truth: dict | None = None) -> dict[str, str]:
        cache = UPSTREAM_CACHE / platform
        if not cache.is_dir():
            self.skipTest(f"no cached original for {platform}")
        scraped = load_platform_config(platform, str(REPO_ROOT / "platforms"))
        systems, report = build_native_model(truth or {"systems": {}}, scraped)
        exporter = discover_exporters()[platform]()
        originals: dict[str, str] = {}
        for path in cache.rglob("*"):
            if not path.is_file():
                continue
            relative = str(path.relative_to(cache))
            if relative.endswith((".zip", ".tar.gz")) and hasattr(exporter, "unpack"):
                originals.update(exporter.unpack(path.read_bytes()))
            else:
                originals[relative] = path.read_text(
                    encoding="utf-8", errors="replace"
                )
        produced = exporter.render(systems, report, originals, scraped)
        self.assertEqual(exporter.validate(systems, produced), [])
        return produced, originals

    def test_retrodeck_keeps_every_bios_entry_it_declares(self):
        """The component list is corrected in place, never replaced.

        Assigning our entries wholesale dropped 114 distinct filenames that
        RetroDECK's own manifests declare.
        """
        produced, originals = self._export("retrodeck")

        def names(text: str) -> set[str]:
            manifest = json.loads(text)
            for value in manifest.values():
                if not isinstance(value, dict):
                    continue
                listing = value.get("bios")
                if listing is None:
                    for key in ("preset_actions", "cores"):
                        nested = value.get(key)
                        if isinstance(nested, dict) and "bios" in nested:
                            listing = nested["bios"]
                            break
                if listing is None:
                    continue
                return {
                    str(e.get("filename", ""))
                    for e in listing
                    if isinstance(e, dict)
                }
            return set()

        lost: set[str] = set()
        compared = 0
        for path, text in produced.items():
            if path not in originals:
                continue
            compared += 1
            lost |= names(originals[path]) - names(text)
        self.assertGreater(compared, 0, "no component manifest was compared")
        self.assertEqual(
            lost, set(), "the export dropped entries RetroDECK declares"
        )

    def test_recalbox_keeps_every_system_and_every_path(self):
        produced, originals = self._export("recalbox")
        before = parse_untrusted_xml(originals["es_bios.xml"], "es_bios.xml")
        after = ET.fromstring(produced["es_bios.xml"])
        self.assertEqual(
            {s.get("platform") for s in before.iter("system")}
            - {s.get("platform") for s in after.iter("system")},
            set(),
        )
        self.assertEqual(
            {b.get("path") for b in before.iter("bios")}
            - {b.get("path") for b in after.iter("bios")},
            set(),
        )

    def test_the_recalbox_export_satisfies_the_schema_recalbox_ships(self):
        schema_path = UPSTREAM_CACHE / "recalbox" / "es_bios.xsd"
        if not schema_path.exists():
            self.skipTest("es_bios.xsd not cached")
        required = self._required_attributes(schema_path.read_text())
        produced, _ = self._export("recalbox")
        root = ET.fromstring(produced["es_bios.xml"])
        for tag, attributes in required.items():
            for element in root.iter(tag):
                for attribute in attributes:
                    with self.subTest(tag=tag, attribute=attribute):
                        self.assertTrue(element.get(attribute))

    @staticmethod
    def _required_attributes(schema: str) -> dict[str, set[str]]:
        """Read the required attributes straight out of the platform's XSD.

        Scoped to each element's own type: a nested declaration belongs to
        the child, and folding it into the parent would demand md5 on
        <system>.
        """
        namespace = "{http://www.w3.org/2001/XMLSchema}"
        root = parse_untrusted_xml(schema, "es_bios.xsd")
        required: dict[str, set[str]] = {}

        def own_attributes(node: ET.Element) -> set[str]:
            found: set[str] = set()
            for child in node:
                if child.tag == f"{namespace}element":
                    continue
                if child.tag == f"{namespace}attribute":
                    if child.get("use") == "required" and child.get("name"):
                        found.add(child.get("name"))
                    continue
                found |= own_attributes(child)
            return found

        for element in root.iter(f"{namespace}element"):
            name = element.get("name")
            if not name:
                continue
            names = own_attributes(element)
            if names:
                required[name] = names
        return required

    def test_batocera_keeps_every_system_and_still_runs(self):
        produced, originals = self._export("batocera")
        before = set(
            re.findall(r'^\s{4}"([^"]+)":\s*\{\s*"name"', originals["batocera-systems"],
                       re.M)
        )
        after = set(
            re.findall(r'^\s{4}"([^"]+)":\s*\{\s*"name"',
                       produced["batocera-systems"], re.M)
        )
        self.assertEqual(before - after, set())

        with tempfile.TemporaryDirectory() as tmp:
            script = Path(tmp) / "batocera-systems"
            script.write_text(produced["batocera-systems"], encoding="utf-8")
            result = subprocess.run(
                [sys.executable, str(script), "--createReadme"],
                capture_output=True,
                text=True,
                timeout=120,
            )
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertTrue(result.stdout.strip())

    def test_batocera_keeps_the_zip_member_checks(self):
        """checkInsideZip verifies a ROM inside the archive, not the archive."""
        produced, originals = self._export("batocera")
        self.assertEqual(
            produced["batocera-systems"].count("zippedFile"),
            originals["batocera-systems"].count("zippedFile"),
        )

    def test_rocknix_still_runs(self):
        produced, _ = self._export("rocknix")
        with tempfile.TemporaryDirectory() as tmp:
            script = Path(tmp) / "rocknix-systems"
            script.write_text(produced["rocknix-systems"], encoding="utf-8")
            result = subprocess.run(
                [sys.executable, str(script), "--createReadme"],
                capture_output=True,
                text=True,
                timeout=120,
            )
        self.assertEqual(result.returncode, 0, result.stderr)

    def test_emudeck_keeps_every_check_and_still_parses(self):
        produced, originals = self._export("emudeck")
        pattern = re.compile(r"^(\w+)\(\)\{", re.M)
        before = set(pattern.findall(originals["checkBIOS.sh"]))
        after = set(pattern.findall(produced["checkBIOS.sh"]))
        self.assertEqual(before - after, set())

        with tempfile.TemporaryDirectory() as tmp:
            script = Path(tmp) / "checkBIOS.sh"
            script.write_text(produced["checkBIOS.sh"], encoding="utf-8")
            result = subprocess.run(
                ["bash", "-n", str(script)], capture_output=True, text=True, timeout=60
            )
        self.assertEqual(result.returncode, 0, result.stderr)

    def test_emudeck_keeps_every_md5_it_declares(self):
        """An md5 the platform holds is never withdrawn from its array.

        The array was assigned wholesale, so any value our model did not
        carry disappeared and a user whose dump matched it stopped passing
        EmuDeck's own check.
        """
        produced, originals = self._export("emudeck")
        pattern = re.compile(r"local\s+hashes=\(([^)]*)\)")
        before = pattern.findall(originals["checkBIOS.sh"])
        after = pattern.findall(produced["checkBIOS.sh"])
        self.assertEqual(len(before), len(after), "an md5 array vanished")
        for index, (was, now) in enumerate(zip(before, after)):
            with self.subTest(array=index):
                self.assertEqual(
                    set(was.split()) - set(now.split()),
                    set(),
                    "the export withdrew an md5 EmuDeck declares",
                )

    def test_retrobat_keeps_every_system_and_every_file(self):
        produced, originals = self._export("retrobat")
        before = json.loads(originals["batocera-systems.json"])
        after = json.loads(produced["batocera-systems.json"])
        self.assertEqual(set(before) - set(after), set())
        self.assertEqual(
            {(s, f["file"]) for s, d in before.items() for f in d["biosFiles"]}
            - {(s, f["file"]) for s, d in after.items() for f in d["biosFiles"]},
            set(),
        )

    def test_retroarch_keeps_every_rom_name_the_dat_declares(self):
        produced, originals = self._export("retroarch")
        pattern = re.compile(r"^\trom \( name (\"[^\"]+\"|\S+)", re.M)
        before = {n.strip('"') for n in pattern.findall(originals["System.dat"])}
        after = {n.strip('"') for n in pattern.findall(produced["System.dat"])}
        self.assertEqual(before - after, set())

    def test_the_dat_keeps_the_system_names_libretro_writes(self):
        """The comment is the name the DAT spells, separator included."""
        produced, originals = self._export("retroarch")
        comments = re.compile(r'^\tcomment "([^"]+)"', re.M)
        before = set(comments.findall(originals["System.dat"]))
        after = set(comments.findall(produced["System.dat"]))
        self.assertEqual(before - after, set())

    def test_the_dat_never_states_a_rom_it_cannot_hash(self):
        produced, _ = self._export("retroarch")
        for line in re.findall(r"^\trom \( (.*) \)$", produced["System.dat"], re.M):
            with self.subTest(line=line[:60]):
                self.assertTrue(
                    any(f" {h} " in line for h in ("crc", "md5", "sha1"))
                )

    def test_misterfpga_keeps_the_download_urls_it_installs_from(self):
        produced, _ = self._export("misterfpga")
        database = json.loads(produced["bios_db.json"])
        self.assertTrue(database.get("db_id"))
        for path, entry in database["files"].items():
            with self.subTest(path=path):
                self.assertTrue(entry.get("url"))

    def test_bizhawk_stays_compilable(self):
        produced, originals = self._export("bizhawk")
        source = produced["FirmwareDatabase.cs"]
        self.assertEqual(source.count("{"), source.count("}"))
        self.assertEqual(
            source.count("FirmwareAndOption("),
            originals["FirmwareDatabase.cs"].count("FirmwareAndOption("),
        )

    # A core RetroPie packages, with a file its help does not yet name.
    RETROPIE_TRUTH = {
        "systems": {
            "magnavox-odyssey2": {
                "files": [
                    {
                        "name": "c52.bin",
                        "required": True,
                        "md5": "f1071cdb0b6b10dde94d3bc8a6146387",
                        "_cores": ["o2em"],
                    }
                ]
            }
        }
    }

    def test_retropie_keeps_every_name_its_packages_already_list(self):
        """RetroPie's BIOS list is prose; a name it writes is never removed."""
        produced, originals = self._export("retropie", self.RETROPIE_TRUTH)
        exporter = discover_exporters()["retropie"]()
        help_of = re.compile(r'rp_module_help="((?:[^"\\]|\\.)*)"')
        for relative, text in produced.items():
            with self.subTest(module=relative):
                before = help_of.search(originals[relative])
                after = help_of.search(text)
                self.assertIsNotNone(after, "the help string is not closed")
                self.assertEqual(
                    exporter._listed(before.group(1))
                    - exporter._listed(after.group(1)),
                    set(),
                )
                self.assertIn("BIOS", after.group(1))

    def test_a_patched_scriptmodule_still_parses_as_bash(self):
        produced, _ = self._export("retropie", self.RETROPIE_TRUTH)
        self.assertTrue(produced)
        with tempfile.TemporaryDirectory() as tmp:
            for relative, text in produced.items():
                script = Path(tmp) / Path(relative).name
                script.write_text(text, encoding="utf-8")
                result = subprocess.run(
                    ["bash", "-n", str(script)],
                    capture_output=True,
                    text=True,
                    timeout=60,
                )
                with self.subTest(module=relative):
                    self.assertEqual(result.returncode, 0, result.stderr)

    def test_retropie_never_drafts_a_sentence_nobody_wrote(self):
        """A package with no BIOS list is a docs change, not a correction."""
        produced, originals = self._export("retropie", self.RETROPIE_TRUTH)
        for relative in produced:
            with self.subTest(module=relative):
                self.assertIn("BIOS", originals[relative])

    def test_retrodeck_writes_a_manifest_per_component(self):
        produced, _ = self._export("retrodeck")
        self.assertTrue(produced)
        for path, content in produced.items():
            with self.subTest(path=path):
                self.assertTrue(path.endswith("component_manifest.json"))
                manifest = json.loads(content)
                component = next(iter(manifest.values()))
                self.assertTrue(
                    component.get("name") or component.get("system"),
                    "the component lost its identity",
                )


if __name__ == "__main__":
    unittest.main()