feat: record each export change before and after

The page named the entries a change touched and never the values, so
a maintainer could not check a correction. The report now carries
changes, one object per change with kind, entry, field, old and new,
read from the platform entry and from the value the render writes
(written() per format: record value for System.dat and RomM, one md5
where a single one is checked). The existing label lists are derived
from it. The page writes "md5 X becomes Y" and names each required
flag with its new value. A test parses the original and updated
System.dat and es_bios.xml and checks the list equals the diff.

scripts/export_native.py

@@ -429,26 +429,89 @@ def refresh_cache(
     return True, [f"{len(wanted)} file(s) fetched"]
 
 
-@dataclass
-class LandedChanges:
-    """What an export changed in the platform's file, field by field."""
-
-    applied: list[str] = field(default_factory=list)
-    filled: list[str] = field(default_factory=list)
-    resized: list[str] = field(default_factory=list)
-    unsized: list[str] = field(default_factory=list)
-    cleared: list[str] = field(default_factory=list)
-    requirements: int = 0
-
-
 def _count(number: int, noun: str, plural: str = "") -> str:
     """"1 hash", "2 hashes": a count with its noun in the right number."""
     plural = plural or noun + ("es" if noun.endswith(("sh", "ch", "s", "x")) else "s")
     return f"{number} {noun if number == 1 else plural}"
 
 
+@dataclass
+class LandedChanges:
+    """What an export changed in the platform's file, field by field.
+
+    `records` holds each change with its value before and after, so a
+    maintainer can check it against the diff. The lists of labels are read
+    from it, for the summary and the page.
+    """
+
+    records: list[dict] = field(default_factory=list)
+
+    def _labels(self, kind: str) -> list[str]:
+        return [
+            f"{r['system']}/{r['name']} {r['field']}"
+            for r in self.records if r["kind"] == kind
+        ]
+
+    @property
+    def applied(self) -> list[str]:
+        return self._labels("hash_corrected")
+
+    @property
+    def filled(self) -> list[str]:
+        return self._labels("hash_filled")
+
+    @property
+    def cleared(self) -> list[str]:
+        return self._labels("hash_cleared")
+
+    @property
+    def resized(self) -> list[str]:
+        return [label.removesuffix(" size") for label in self._labels("size_corrected")]
+
+    @property
+    def unsized(self) -> list[str]:
+        return [label.removesuffix(" size") for label in self._labels("size_removed")]
+
+    @property
+    def requirements(self) -> int:
+        return sum(r["kind"] == "requirement_corrected" for r in self.records)
+
+
+def _change(
+    kind: str, entry: object, field_name: str | None, old: object, new: object
+) -> dict:
+    return {
+        "kind": kind,
+        "system": entry.native_system,
+        "name": entry.name,
+        "destination": entry.destination,
+        "field": field_name,
+        "old": old,
+        "new": new,
+    }
+
+
+def _platform_value(entry: object, field_name: str) -> str:
+    return ",".join(_hash_values(entry.platform or {}, field_name))
+
+
+def _added_fields(exporter: object, entry: object) -> dict:
+    """The fields an addition is written with, as the render chooses them."""
+    carried = exporter.carries()
+    written: dict[str, object] = {
+        name: value
+        for name in HASH_FIELDS
+        if name in carried and (value := exporter.written(entry, name))
+    }
+    if "size" in carried and entry.size() is not None:
+        written["size"] = entry.size()
+    if "required" in carried:
+        written["required"] = entry.required
+    return written
+
+
 def landed_changes(exporter: object, systems: dict) -> LandedChanges:
-    """The changes the written file carries.
+    """The changes the written file carries, with their values.
 
     Only what the format states: a correction to a field the file has no
     place for, or one its render keeps out, is not a change the maintainer
@@ -456,26 +519,39 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
     did not do. A hash written where the platform left none changes the
     check from existence to content, so it is counted too. The truth taking
     over an entry also rewrites its size and drops the platform's hashes it
-    does not declare: both are in the diff.
+    does not declare: both are in the diff. An addition carries the fields
+    it is written with as its new value.
     """
     changes = LandedChanges()
+    records = changes.records
     for system in systems.values():
         for entry in system.files:
-            label = f"{entry.native_system}/{entry.name}"
+            if entry.platform is None:
+                if entry.name and exporter.lands(entry):
+                    records.append(_change(
+                        "added", entry, None, None, _added_fields(exporter, entry)
+                    ))
+                continue
             for field_name in entry.corrections:
                 if not exporter.states(entry, field_name):
                     continue
                 if field_name == "required":
-                    changes.requirements += 1
+                    records.append(_change(
+                        "requirement_corrected", entry, "required",
+                        bool(entry.platform.get("required")), entry.required,
+                    ))
                 else:
-                    changes.applied.append(f"{label} {field_name}")
-            changes.filled.extend(
-                f"{label} {field_name}"
+                    records.append(_change(
+                        "hash_corrected", entry, field_name,
+                        _platform_value(entry, field_name),
+                        exporter.written(entry, field_name),
+                    ))
+            records.extend(
+                _change("hash_filled", entry, field_name, None,
+                        exporter.written(entry, field_name))
                 for field_name in entry.filled
                 if exporter.fills(entry, field_name)
             )
-            if entry.platform is None:
-                continue
             theirs = entry.platform.get("size")
             ours = entry.size()
             if (
@@ -484,7 +560,9 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
                 and _single_size(theirs) not in (None, ours)
                 and exporter.states(entry, "size")
             ):
-                changes.resized.append(label)
+                records.append(_change(
+                    "size_corrected", entry, "size", _single_size(theirs), ours
+                ))
             elif (
                 theirs
                 and ours is None
@@ -493,9 +571,12 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
             ):
                 # The truth took the entry over without a size of its own:
                 # System.dat's neocdz.zip line went out without size 953078.
-                changes.unsized.append(label)
-            changes.cleared.extend(
-                f"{label} {field_name}"
+                records.append(_change(
+                    "size_removed", entry, "size", _single_size(theirs), None
+                ))
+            records.extend(
+                _change("hash_cleared", entry, field_name,
+                        _platform_value(entry, field_name), None)
                 for field_name in HASH_FIELDS
                 if _hash_values(entry.platform, field_name)
                 and not entry.hashes(field_name)
@@ -503,7 +584,10 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
             )
     # A value the render rewrote though the model reads it as the
     # platform's own (RomM's sha1 with a leading space).
-    changes.applied.extend(getattr(exporter, "normalized", []))
+    records.extend(
+        _change("hash_corrected", entry, field_name, old, new)
+        for entry, field_name, old, new in getattr(exporter, "normalized_values", [])
+    )
     return changes
 
 
@@ -661,6 +745,7 @@ def export_platform(
             "not_added": refused,
             "not_kept": lost,
         },
+        "changes": changes.records,
         "hashes_corrected": applied,
         "hashes_filled": filled,
         "sizes_corrected": resized,

scripts/exporter/base_exporter.py

@@ -180,6 +180,17 @@ class BaseExporter(ABC):
             return ""
         return fe.hash(field_name)
 
+    def written(self, fe: NativeFile, field_name: str) -> str:
+        """The value the render writes for one hash field, comma-joined.
+
+        The report gives each change its value before and after, so it reads
+        the value the way the render chooses it: one md5 where the format
+        checks one, every accepted value where it lists them.
+        """
+        if field_name == self.SINGLE_HASH_FIELD:
+            return self.single_value(fe, field_name)
+        return ",".join(fe.hashes(field_name))
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         """Whether the written file carries this entry's corrected field.
 

scripts/exporter/misterfpga_exporter.py

@@ -64,6 +64,10 @@ class Exporter(BaseExporter):
             return False
         return super().states(fe, field_name)
 
+    def written(self, fe: NativeFile, field_name: str) -> str:
+        """An entry is checked against one md5, the first accepted."""
+        return fe.hash(field_name)
+
     def _by_path(self, systems: dict[str, NativeSystem]) -> dict[str, object]:
         indexed: dict[str, object] = {}
         for system in systems.values():

scripts/exporter/retrodeck_exporter.py

@@ -90,6 +90,12 @@ class Exporter(BaseExporter):
             return False
         return super().writable(fe, require)
 
+    def written(self, fe: NativeFile, field_name: str) -> str:
+        """md5 is a list of every accepted dump, sha256 a single value."""
+        if field_name == "sha256":
+            return fe.hash(field_name)
+        return super().written(fe, field_name)
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         if id(fe) in getattr(self, "_appended", ()):
             return False

scripts/exporter/romm_exporter.py

@@ -113,6 +113,7 @@ class Exporter(BaseExporter):
         }
         self._first_addition = {}
         self.normalized: list[str] = []
+        self.normalized_values: list[tuple[NativeFile, str, str, str]] = []
         for system in sorted(systems.values(), key=lambda s: s.native_id):
             for fe in sorted(system.files, key=lambda f: f.name):
                 key = (fe.native_system, fe.name)
@@ -159,6 +160,10 @@ class Exporter(BaseExporter):
         does not keep."""
         return bool(_hash_values(fe.platform or {}, name)) and not fe.hashes(name)
 
+    def written(self, fe: NativeFile, field_name: str) -> str:
+        """A record holds one value per field, the one record_hash names."""
+        return fe.record_hash(field_name)
+
     def fills(self, fe: NativeFile, field_name: str) -> bool:
         return self.states(fe, field_name) and self.record_fills(fe, field_name)
 
@@ -179,6 +184,7 @@ class Exporter(BaseExporter):
             and name not in fe.corrections
         ):
             self.normalized.append(f"{fe.native_system}/{fe.name} {name}")
+            self.normalized_values.append((fe, name, str(theirs), value))
         entry[field] = value
 
     def validate(

scripts/exporter/systemdat_exporter.py

@@ -154,6 +154,10 @@ class Exporter(BaseExporter):
         )
         return lines
 
+    def written(self, fe: NativeFile, field_name: str) -> str:
+        """A record holds one value per field, the one record_hash names."""
+        return fe.record_hash(field_name)
+
     def fills(self, fe: NativeFile, field_name: str) -> bool:
         return self.states(fe, field_name) and self.record_fills(fe, field_name)
 

scripts/siteexports.py

@@ -27,6 +27,60 @@ CHANGE_KINDS = (
 )
 
 
+# Report key, then the kind each line of `changes` carries.
+CHANGE_RECORDS = {
+    "hashes_corrected": "hash_corrected",
+    "hashes_filled": "hash_filled",
+    "sizes_corrected": "size_corrected",
+    "sizes_removed": "size_removed",
+    "hashes_cleared": "hash_cleared",
+}
+
+
+def _value(value: object) -> str:
+    return f"`{value}`"
+
+
+def describe_change(change: dict) -> str:
+    """One change in words, with the value before and after it."""
+    label = f"`{change['system']}/{change['name']}` {change['field']}"
+    kind, old, new = change["kind"], change["old"], change["new"]
+    if kind in ("hash_corrected", "size_corrected"):
+        return f"{label} {_value(old)} becomes {_value(new)}"
+    if kind == "hash_filled":
+        return f"{label} {_value(new)} added"
+    if kind == "requirement_corrected":
+        return f"`{change['system']}/{change['name']}` becomes {'required' if new else 'optional'}"
+    return f"{label} {_value(old)} no longer listed"
+
+
+def change_items(report: dict, key: str) -> list[str]:
+    """The lines a kind of change shows. A report written before values
+    were recorded names the entries only."""
+    kind = CHANGE_RECORDS.get(key)
+    changes = report.get("changes")
+    if kind and changes is not None:
+        return [describe_change(c) for c in changes if c["kind"] == kind]
+    items = report.get(key) or []
+    return [str(i) if key == "withheld" else f"`{i}`" for i in items]
+
+
+def requirement_lines(report: dict) -> list[str]:
+    """Each required flag aligned with the code, and what it is now."""
+    items = [
+        describe_change(c) for c in report.get("changes") or []
+        if c["kind"] == "requirement_corrected"
+    ]
+    if not items:
+        return []
+    return [
+        f'??? note "Required flags aligned with the code ({len(items)})"',
+        "",
+        *[f"    - {item}" for item in items],
+        "",
+    ]
+
+
 def _plural(count: int, noun: str, plural: str = "") -> str:
     return f"{count:,} {noun if count == 1 else plural or noun + 's'}"
 
@@ -241,14 +295,15 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
             (name, url), = originals.items()
             lines.extend([f"Based on [`{name}`]({url}).", ""])
 
+        lines.extend(requirement_lines(report))
         for key, title in CHANGE_KINDS:
-            items = report.get(key) or []
+            items = change_items(report, key)
             if not items:
                 continue
             lines.extend([
                 f'??? note "{title} ({len(items)})"',
                 "",
-                *[f"    - {item}" if key == "withheld" else f"    - `{item}`" for item in items],
+                *[f"    - {item}" for item in items],
                 "",
             ])
     return "\n".join(lines)