From a9c175d32277e5664eedaf776b1f55939662b212 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:44:42 +0200 Subject: [PATCH] chore: regenerate the updated platform files --- exports/misterfpga.json | 3 +- exports/misterfpga/bios_db.json.zip | Bin 0 -> 5015 bytes scripts/export_native.py | 10 ++++-- scripts/exporter/base_exporter.py | 9 ++++++ scripts/exporter/misterfpga_exporter.py | 34 +++++++++++++++++++-- tests/test_exporters.py | 39 ++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 exports/misterfpga/bios_db.json.zip diff --git a/exports/misterfpga.json b/exports/misterfpga.json index 0aa1df62..ea1da06f 100644 --- a/exports/misterfpga.json +++ b/exports/misterfpga.json @@ -4,7 +4,8 @@ "bios_db.json.zip": "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/7cfa7a3e84af078d0efba74fa46c50357c4f567c/bios_db.json.zip" }, "files": [ - "bios_db.json" + "bios_db.json", + "bios_db.json.zip" ], "summary": "81 kept, 0 added, 0 hashes corrected, 0 requirements corrected", "counts": { diff --git a/exports/misterfpga/bios_db.json.zip b/exports/misterfpga/bios_db.json.zip new file mode 100644 index 0000000000000000000000000000000000000000..bd0b97e09aa2d74c7e1e1ec7ab7937a586650a25 GIT binary patch literal 5015 zcmV;I6KL#EO9KQH0000805EtxUH0U-ejO740Jd8I01N;C0Aguxb6;d)E^2dcZp~b2 zbK6GJ{wpq4`P)jG``B+1ZELlOj|vm7-+F6HHK&XXMX*Rpu{V|fzC8mTAPzAwX~(IU zA_i<=o|&HRryD)^edO(J;YWXeHnK&zy!CeUw`ZeROv4h|ejnYL`>?#AFWz3*vRs`n z%KO~>(JVhEnaCs&%zGwdpeRaeEOiJYgt^0<=Zlq(Ve#9dSOpM>X1WZ}HaDv*KM$W) z={L*bzv(naOmQioVP)=82mYUszda)q{;~A$-*Dz(F-?*mSF8E*?-v(l;XW3R;k;bj zUHJ01*|aqN;=Wj}!s2#~D0Tn*zr}n9;SadZ4%x3}_Mf5!%@G8N|)a^(4dG9z7?oI%czh7`Gr3WDupyF(#cwLL0=83ET$kHB~+iom7gEB;OV? zZem`p7Ns*jbWgJMgc~9PcT6#CsbDUGN@GcRFcHy#h!xhIc*QYzHwZG`D>I#beooN4 zMH$0#S(Gy~g~Q85`o(iHPTw~I@clWF6mt!K;Na@!?IgEBH>G+`pjmCM_d4F4!=G`v zn8ROvZi`uc6jx?Z{P=nHsmIPa&k*%i=*St-$a0J*_liUX<0yyPIgUFAQ64jh8vuQl zZv5h`$nB_w&N$?mP+--BkWxGCqoC9Vt-K)|nW5+iBT97v)JFhJ{oi6kec;+lo_c~@ zl$@i8tt6l<$}(bb^xPUJ$cYxf$)GjY_ZAvyLDNEmDH9@UE5P4sjf?_}Mi7NK6N9~P zuQfXvIMM|7cKnw1XeG=Qb;7}Ni_`_|B-JE>pVE{DWW`WhO^Fm;N=eZ%;L34H`?Ye5 zuC12J8B0`T7R(p6Dy=w&q_xy?Z9z9rqn0NG*V4-^xVBb^d>mR=fgJh$)OpAm#mkA8BlI75y*5v3GfDWFp_k_eb%aB47@5L9)tCv^XU}F(D6z>fm>i-$i ztspL`EIDohq5IYRRY>mgpW6-^mnM5$-gjfAkzlF3!~l&V2<{{!%rhOLCR)j%CaE`~ zqx4911q2(o={xZ9m!IG_71iId_#cYZqL_VxgGY3ZXq&bt`5%heUH5Rn6bI@om?>tW za)E{zv67Bq&OJ6mL-<}^OOIW>2|}gc_XMj3#+%tHOsB=;&Mvooksv{e6p}b#f>I+2 zM+%|H7?oC?%;6pr)$YN_U|)U;?qLOTFu!1~s@U%5kD(dCy*4WlZBKrCVA|FjInOl{ z;Ory~j06>!BbZ95I2;aN5r}mvz-b5O`l->l4zsYx>*Qtzaqz=^n5~w7W^mdx&2yfT zb`bjE;6y;cu8B7g9i;?;8%kkOO<>Y33#F|?e zx9Xu1Mn{VR#HG?8q=71GucV+Hdk?XWI*~#hlWf`m*y>U`$ZTmJ=s(TfgIRRL6hSd2 zxt0Wxlv-1;kb*OjQh}7gLvw1qaW_3CAyfg#FsrZKDbpIxw)Mm+BFV7zR#~g9fDcVI zxKxc85oYjW=;ae%186f~ts!ljLLdm>F0ms*Y2_kf5ZnbUH1&i>h*`huDQ!ltKbGbE za}QJSoB$~b9^0s`M?yd_2QCp2Be>(%pDL?zbkv$b^rlvJ5ZhYo(nKn)opuI6R4S-P zOn8BS`b&(5LhPq~C&t)R*w>Q}xW~?dei8^wxp6^y5}5YLC`5qh2aohnZS9*xb_}mg zz0c4d@1SSs)&gL6md>_tn#Jn##eE9Q@2O~bV?(?MSQ!x!0DwGULdQw^({W%6)eNo)UbdFc6(;jtEthYAFfUnh4>(bC9wQ zGTfQ*i;I^Ji*g<=oUdZ_R@&Nfh*xYRpsL(P43yJ&>#$Nday(djTeg;035SH19g|6_ z&}6L7`Bf&G{6%__1np-ssluCyQ#yPk=LjY0bnn|^^G`E3v)-W*iW+5fAd(6} zI2dP1fS&tkh~g?7ozbrk%`q8^KRq-V?*7P(`-bM1$z|`r01%^*!eHz?C=SKKSPBY5 zF%^>Vp=;#HfjK6*(Kj#?XivPnHZ%bDpzZi9KNA=bCn7!iS(5vFBwRm?l&0z1)w z9F?`J4aP5%?m_q|+(8Tj-(M}ldIf2U1cFEoO>avGaze2%4Q~EMudYT;*yFuUkp3rV!b>yHKp~O24v8Vyg#*qP{Dj0A%h}j|? zY%#wwuBL?Lz+N!piItXEgvrq2udfcVhU4t=<>WTG z);8K@EIys5kH$(QmU~c7&`z#mbjV9!+)RO)Q52n|PU4hUcPQEb=VwdOnoOdFid;v= zY^1O#sWwJxs<8;V?ybAU3!f?aDi| zV{#U$`@I>NEw&E&qGU<~I|dq=IE?m8g-AIuG)6>*CS<-owibcOstbWlGDFU1I=bgC zsOFNGD7j+Ja!aE`CMf3=Fi|@=PuiJr%Ioi#%y$K1&Bu2*x=^l4SfF# zgaJbf%3%0U1QDWk!};|$z*$GKL;7wECZs2~7BNImo<~I`mJp0+MFLOtgBVC$q;6<# zO;dt9we^^yx&$EOtB|C~`-hvG6k`3fekRt?;CGV`J#l0Z$`Oh^Qd(FUjE4{dgMU=% ziSl3`(3>pf$3)f8`;2xne)kWF5I(=UdD8)|qslHYhgsx8N(L?>l86{mP64n83w9`x z`zGj654!e43#&a4Q}FkWQtw2nZk5(z>O2$ChoLO|o1h~d=$1BDi0)yNC}r$TD&Fum zde5m1;O(`uiW?jUCq=Q+2}aA*oCMnjKn+&uehsVfM>Cy<*sDWgWDlkBZ4rM)k zb9+o0w}zqeCcES3@1m>Qjos8f2>^J2q9R>`4{&_(xiWqu5gfWEH{*LxU@{E0d=e{y|wF?pDW#j{k& z_3Wzr{0u^y=o)^=c}atXSS8BfWl#{BMQnpY2z>9*YVVe%*1Poem*c&@s_Tz=|#VkKO}(nhXybT%3hl?U?- z^gd!82lM;(UO;?;*mJT8KCdQcANDGvUe4~O#q#6&X#>A{lUDJIpI%oh8yX10A{ANU za|&(Ep)sq6MNdIEqDVLtfKz!P6!D&TB_KGko*5^^(5SVafgFIqpTEL#zJhar zIH~g+^O%g-KZ?ys;FpLtB8JK8!58J(%}r1EZmA?FJTRqLNW=1vn)eGD9bA&!7c+fD$Q+T|W)5y2b=wYuZBhXfZvs|~BPJ}3BpL`63|J^}WU~KT zp>C@8&)2U$V3BgHz1305Gis2Q1`BQ>P6XzOqUVknPaHXV4-Y*7)%^KkVaw}CFiVpPRfR#|i@l3XN;EY8inS1^ z(>WdUv|xjwmrA`ad?>FLrsyHg0mIrAI5}Pw2z7=#K`kRpLh?V(E<( zhb6#tiuz#W_Z=J*qo^)Y-+h1hj;bzN+r@tPrnn}#-p&4`P~LT3m$|xp^7qvq?z%wU zpM1Du4Vkpp_plIc2&LVzhox9eYHx$wVNu=?Olq; z6$d3)eI#cHKz(6PuCJaH(2ePu9GYL;6v#>E!=2)d5!~ie4vW_9!8T+) zELPXrZP@yx80|0%n@H*K{pvcri#-m@)s=Bag$~LI)G3$w;9+6#xPMe_x4v+z^p?6s zWoM~RwZ!Sgjw6%kP*G&D^hUsusqvij%Djo4@c457cM(#R1Hq$=F{s<@=)0_p_IEL;~&k;RZq3nt(AOwM@T;WwQ}F-B^g@+ zO7ivW=2F&dQQu1A`uvAAD;N>(rDNoLs#q`LFbER=U4&1R~5T_5F3B*O{E%k?nL?&j%X zXil2W*3Ze;r{->LQB%1W9D%hfX4O2U@tJ2k=r(rnG1)<$^B z%}y2crm1#|e7GBj)V8`ysi7_IRiQOYlg@K$CS{NAs_fD3C)>tt)59`}r2Sc=$7CSv z+CZc<$LP1bgqPLQPtkI%#jO{$zBV zz{f7;gA=);_>7@#sgVy$w`{!fI)fwf9$Baj=MT(ULfjF5u7UGlDvZj@i#DiRvrB)hufDji?wlrLV@|V)f<*|HIK9^< zPtyJkvMxWNL4gE^ndBr2ZbEt|ful5)Xhs6xsxt6RWofEnB*Qhx`7=uYybg_DoKlUp zCC0nnw2KOqGs+TgjSzwag$Ro}NR>A4I}zQJ1p{BCOU`#|)4Fqg`m{&1pT71fmFkZ_ z{s&M?0Rj{Q6aWAK2mmm6Jze(XxPBcI006dI000aC0000000000004ji00000Vrg%4 hUu0q~YIARHP)h{{000000RRC2IsgCwM-u=5000p_bh`im literal 0 HcmV?d00001 diff --git a/scripts/export_native.py b/scripts/export_native.py index 98ea7454..629eb03f 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -339,6 +339,9 @@ def collect_originals( continue if archive: originals.update(_read_archive(unpack, payload)) + keep = getattr(exporter, "keep_archive", None) + if callable(keep): + keep(relative, payload) else: originals[relative] = payload.decode("utf-8", errors="replace") # One platform's unreadable file fails that platform: raised, it @@ -760,7 +763,9 @@ def export_platform( return True, messages -def _write_produced(root: Path, produced: dict[str, str], modes: dict[str, int]) -> None: +def _write_produced( + root: Path, produced: dict[str, str | bytes], modes: dict[str, int] +) -> None: """Write each file as its bytes, whole or not at all, with its mode. write_text translated line ends on Windows, so FirmwareDatabase.cs's @@ -771,7 +776,8 @@ def _write_produced(root: Path, produced: dict[str, str], modes: dict[str, int]) for relative, content in produced.items(): path = root / relative path.parent.mkdir(parents=True, exist_ok=True) - write_bytes_atomic(str(path), content.encode("utf-8")) + data = content if isinstance(content, bytes) else content.encode("utf-8") + write_bytes_atomic(str(path), data) path.chmod(modes.get(relative, 0o644)) diff --git a/scripts/exporter/base_exporter.py b/scripts/exporter/base_exporter.py index 31056015..b91289a0 100644 --- a/scripts/exporter/base_exporter.py +++ b/scripts/exporter/base_exporter.py @@ -126,6 +126,15 @@ class BaseExporter(ABC): """ return True + def keep_archive(self, relative: str, payload: bytes) -> None: + """Hold the bytes of an original archive the render may hand back.""" + if not hasattr(self, "_archives"): + self._archives: dict[str, bytes] = {} + self._archives[relative] = payload + + def archive(self, relative: str) -> bytes | None: + return getattr(self, "_archives", {}).get(relative) + @staticmethod def file_modes() -> dict[str, int]: """The mode of each file the platform publishes other than 0644.""" diff --git a/scripts/exporter/misterfpga_exporter.py b/scripts/exporter/misterfpga_exporter.py index 7869a336..0d0df758 100644 --- a/scripts/exporter/misterfpga_exporter.py +++ b/scripts/exporter/misterfpga_exporter.py @@ -21,6 +21,27 @@ SOURCE_URL = ( _DB_NAME = "bios_db.json" +_ARCHIVE = "bios_db.json.zip" + + +def rebuild_archive(original: bytes, member: str, text: str) -> bytes: + """The archive with one member's content replaced, as MiSTer writes it. + + The member keeps its name, date, compression and attributes, so the + archive differs from MiSTer's only by the content that changed. + """ + with zipfile.ZipFile(io.BytesIO(original)) as source: + info = source.getinfo(member) + rebuilt = zipfile.ZipInfo(info.filename, date_time=info.date_time) + rebuilt.compress_type = info.compress_type + rebuilt.external_attr = info.external_attr + rebuilt.create_system = info.create_system + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as target: + target.writestr(rebuilt, text.encode("utf-8")) + return buffer.getvalue() + + class Exporter(BaseExporter): """Write MiSTer's bios_db.json, corrected.""" @@ -82,7 +103,7 @@ class Exporter(BaseExporter): report: Report, originals: dict[str, str], scraped: dict | None = None, - ) -> dict[str, str]: + ) -> dict[str, str | bytes]: original = originals.get(_DB_NAME) or originals.get("bios_db.json.zip") if not original: raise ValueError( @@ -111,7 +132,16 @@ class Exporter(BaseExporter): # MiSTer publishes the database as one line, without a final # newline: written with an indent it came out as 1499 changed lines. - return {_DB_NAME: json.dumps(database, ensure_ascii=False)} + text = json.dumps(database, ensure_ascii=False) + produced: dict[str, str | bytes] = {_DB_NAME: text} + # MiSTer's Downloader fetches the archive, not the JSON in it: handed + # back as it came when nothing changed, rebuilt around the new + # member otherwise. + archive = self.archive(_ARCHIVE) + if archive is not None: + unchanged = text == originals.get(_DB_NAME) + produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text) + return produced def validate( self, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 47585e76..402b2e66 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1546,6 +1546,45 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase): """The BiosDB is one line of JSON. Written with indent=2 it came out as 1499 changed lines for a database whose content had not changed.""" + def _render(self, truth: dict): + archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" + if not archive.exists(): + self.skipTest("upstream cache not populated") + raw = archive.read_bytes() + exporter = discover_exporters()["misterfpga"]() + exporter.keep_archive("bios_db.json.zip", raw) + originals = exporter.unpack(raw) + config = native_layer(load_platform_config("misterfpga", str(REPO_ROOT / "platforms"))) + systems, report = build_native_model(truth, config) + return raw, exporter.render(systems, report, originals, config) + + def test_the_archive_mister_fetches_is_published_as_it_came(self): + """The Downloader fetches bios_db.json.zip, and the export gave only + the JSON inside it.""" + raw, produced = self._render({"systems": {}}) + self.assertEqual(produced["bios_db.json.zip"], raw) + + def test_a_rebuilt_archive_keeps_the_member_as_mister_writes_it(self): + raw, _ = self._render({"systems": {}}) + from exporter.misterfpga_exporter import rebuild_archive # noqa: PLC0415 + + rebuilt = rebuild_archive(raw, "bios_db.json", '{"changed": true}') + with zipfile.ZipFile(io.BytesIO(raw)) as before, \ + zipfile.ZipFile(io.BytesIO(rebuilt)) as after: + old, new = before.infolist()[0], after.infolist()[0] + self.assertEqual(after.read("bios_db.json"), b'{"changed": true}') + self.assertEqual( + (old.filename, old.date_time, old.compress_type, old.external_attr), + (new.filename, new.date_time, new.compress_type, new.external_attr), + ) + + def test_bytes_are_written_untouched(self): + import export_native # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + export_native._write_produced(Path(tmp), {"a.zip": b"PK\x03\x04\xff"}, {}) + self.assertEqual((Path(tmp) / "a.zip").read_bytes(), b"PK\x03\x04\xff") + def test_an_empty_truth_hands_back_the_database(self): archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" if not archive.exists():