fix: stop git normalising preserved bytes

SHA1 is the primary key of this collection and there was no
.gitattributes, so git guessed. Git for Windows sets core.autocrlf=true
by default: a clone there rewrites every file git considers text,
meaning the shaders, .ini, .txt and .dat assets under bios/ arrive with
CRLF and a different hash from the one published. Verification then
fails on files nobody touched.

bios/ and data/ are exempt from normalisation, generated artefacts are
pinned to LF so a Windows checkout does not show them modified, and the
rule order is asserted rather than assumed.
This commit is contained in:
Abdessamad Derraz committed 2026-08-11 18:52:10 +02:00
1 parent 851b9c9e50
commit d28efae88c
2 files changed
+69

No files matched your search

+23
View File
@@ -0,0 +1,23 @@
# Source and generated artefacts keep LF on every platform, so a Windows
# checkout does not show them as modified and the freshness guard keeps
# meaning something.
*.json text eol=lf
*.yml text eol=lf
*.md text eol=lf
*.py text eol=lf
*.sh text eol=lf
install.ps1 text eol=lf
# Preserved files are bytes, not text, and these rules come last so they win.
#
# Without them git guesses. A clone on Windows, where the installer sets
# core.autocrlf=true by default, rewrites every file git considers text on
# checkout: shaders, .ini, .txt and .dat assets under bios/ come out with CRLF
# line endings and a different SHA1 from the one this repository publishes.
# Verification then fails on files nobody touched, and a re-commit silently
# changes the bytes of a preserved asset.
#
# provenance/ is not listed: those snapshots are generated here, not
# preserved, so they follow the LF rule above.
bios/** -text
data/** -text
+46
View File
@@ -814,6 +814,52 @@ class EveryManifestEntryIsFetchable(unittest.TestCase):
self.assertIn(entry.get("reason"), allowed) self.assertIn(entry.get("reason"), allowed)
class PreservedBytesAreNeverNormalised(unittest.TestCase):
"""git must not rewrite a preserved file's line endings.
SHA1 is the primary key of this collection. Git for Windows sets
core.autocrlf=true by default, so without an attribute saying otherwise a
clone there rewrites every file git guesses is text -- shaders, .ini,
.txt and .dat assets under bios/ -- and each one arrives with a different
hash from the one published here.
"""
def _attr(self, path: str) -> str:
import subprocess
out = subprocess.run(
["git", "check-attr", "text", "--", path],
capture_output=True, text=True, cwd=ROOT,
).stdout
return out.rsplit(":", 1)[-1].strip()
def test_gitattributes_exists(self):
self.assertTrue(
(ROOT / ".gitattributes").is_file(),
"without it git guesses, and guesses wrong on Windows",
)
def test_collection_paths_are_exempt_from_normalisation(self):
for path in (
"bios/Sony/PlayStation/scph5501.bin",
"bios/Other/j2me-loader/color.fsh",
"data/anything.txt",
"data/dolphin-sys/config.json",
):
with self.subTest(path=path):
self.assertEqual(self._attr(path), "unset", f"{path} may be rewritten")
def test_generated_artefacts_stay_lf(self):
for path in (
"database.json",
"scripts/dedup.py",
"README.md",
"provenance/redump.json",
):
with self.subTest(path=path):
self.assertEqual(self._attr(path), "set")
class FreshnessGuardMechanics(unittest.TestCase): class FreshnessGuardMechanics(unittest.TestCase):
"""write_if_changed is what makes `git diff --exit-code` a real check. """write_if_changed is what makes `git diff --exit-code` a real check.