chore: merge audit fixes (scrapers)

This commit is contained in:
Abdessamad Derraz committed 2026-10-21 05:44:32 +02:00
commit f4a225488d
26 files changed
+25412 -119

No files matched your search

+331
View File
@@ -0,0 +1,331 @@
{
"repository": "https://github.com/libretro/libretro-core-info",
"blobs": {
"00_example": "68a87f20d197cfb075dc744b911c354f4ee9c7e3",
"2048": "df66c7f6b23da5b6d7df6ec38ecdded2ea785225",
"3dengine": "f93e2625d5445c4e0319088f4fbacd693a9bf5a6",
"81": "4e0c43e15f0bdc299b2d706a1d496505a8c95ba1",
"DoubleCherryGB": "aedcc6d5a2fbd4df97ce391daab8019134b51384",
"FreeIntvTSOverlay": "469592af9ebe2239b3665218abde6e39d1a9424d",
"a5200": "89680e62646b7a9e1ba04135063cd6e6033d13f9",
"advanced_tests": "a13ebba9e3d4b1f70f882d824268152026ff184a",
"amiarcadia": "73e48e85824950f30df26673219cd5ca4548496a",
"amiberry": "e5d1f0f9a143594d893031caed9aa760ff34ce0a",
"anarch": "f06701967498ab0a5dfc0c7ca50c3aeea6c9e6e6",
"applewin": "c999c4b73ff48409acecb0239214c74f3464bcd1",
"ardens": "938877e6a867441fbab2edfd19acc3d85f4b0277",
"arduous": "cb6f0917fa4454126fdb93714eadb6d6cfab1a9f",
"armsx2": "c8a763b3d75e84997218323350a22cc79fe8e39a",
"atari800": "07b87e91cf4577db7791188399f1cc1f320932f0",
"azahar": "1173e2e8a919d578cc589545cb3c0c71dc4e1188",
"b2": "140b31e2d829f094fbf81856f63acd8f3c97f9e8",
"bbkemu": "ee36e46bfcff3ec6f66da3e35bc249d398d962ca",
"beetle_saturn": "d0364ee2ba9f0ba1e676dcf397cc66aad5204f8d",
"bennugd": "620191828ed5ac08426aa132a7e15dabfe78b702",
"bk": "7025d150c3265eaaabf8c1863a7fe367402f2dc1",
"blastem": "0850a1ce3c92ed115ea42d7df89741c26e46907a",
"bluemsx": "8ea8ac764e50f2618e4936dcf7d8dc43952488c1",
"bnes": "fcd6e352d4949a876014c516d9afc86fd25ae92c",
"boom3": "f7c3290aa476e2fa724b97974713bdfb9d9594d9",
"boytacean": "88da1bdd694a20ed1e3065cab07fe40205cedd9a",
"bsnes": "e414e247241ad1578c7c500b6009d0d9f3793900",
"bsnes-jg": "ea6966241dede9af48e3719478e41f0f8e3af565",
"bsnes2014_accuracy": "f4eb380fb2d70883b6bac33a7f8d6a8fb5a14ef2",
"bsnes2014_balanced": "ac390a8a0c6e6f095f55ce5806f55f02cd599937",
"bsnes2014_performance": "c733738d8b87b75def115381b066271df424affd",
"bsnes_cplusplus98": "f41a72cbba0287cd2d77d0f6d3344fc388b0f4eb",
"bsnes_hd_beta": "587c7282de1309894e49d7e90a6ea646cebf859f",
"bsnes_mercury_accuracy": "ac0794b5c57ab507326f9024ee2a1fd8b4d7667c",
"bsnes_mercury_balanced": "311b1630f18ab748901c5ee1a5a78b1424ba76e9",
"bsnes_mercury_performance": "82478bc1d89f8c09192461ef63119e3c7de33644",
"cannonball": "63d5881e7b6f64c4e320d937d90aec90783325f6",
"cap32": "6c814e8a1bfe4ec17ab4d36a24e6bbc919accf2e",
"cdi2015": "383f3b1388d2181e9af2f147e8c42e156b1ef3ae",
"cemu": "ab3e19d6975762f03a17e6602f504915622e7416",
"chailove": "79e1ee7b89725e4cba1334101b7cd1cae5d3f4f0",
"chimerasnes": "c038f874a3fc5ba64313063a7354f79438edbf61",
"citra": "e5d3b56989a82db718ee1d840571be9f1cc50b37",
"citra2018": "d306c4e2cb65cf83930ba4b5f131a61077a17a78",
"citra_canary": "0b1fd85442193d317370fb5410293366b1595c6c",
"clownmdemu": "00482d6fe4127fcd9fc59df68a0c9b5475c06c9e",
"craft": "e6fa30b75f892dfce039765b1b842e1f970ab038",
"crocods": "e3510a13cf2b285745a467c28ed3bb521f7fe936",
"cruzes": "d5bea3f63b4ad964fc5dc98bca37ce083f9732c1",
"daphne": "2d9268f2857ffbb826faba53a8dab973a2d8e2ff",
"desmume": "b176acacbbbec9d1924cd452fbf749e2515d61d9",
"desmume2015": "c3c14e4185b97115f9fe36ba2efa4f62f8742fec",
"dice": "2f1a4407867e03bb4f1e3a4590b318037ef6b709",
"dingooemu": "cd176c4ababe6feb9f27124a323564e4a13b3950",
"dinothawr": "ec1748f05742ce35cbd084b78785214e7d6a7cc8",
"directxbox": "a22c5e753b66a08aa98529c8145d1a4ccd328a21",
"dirksimple": "9754af3cff83a269fb29d17659bb6770e3b3f536",
"dolphin": "8ab1934d2c309f738f69ef603be9bebacd5fa545",
"dolphin_launcher": "6af4403328719b1af674e34fcb3aabfee1d6dd29",
"dosbox": "b9979165292ec0a4aa9fc0bece5887aeb195112e",
"dosbox_core": "5634e336ccefa587960bf6099ecb247abb879db1",
"dosbox_pure": "2afbf74996a013507f6fcb76a2ed40ca82eddb80",
"dosbox_svn": "e21a8e0ec4912ae4efb203a68231dceaf82b73b5",
"dosbox_svn_ce": "8f06959e2c24f023174e9eeeef7b5818a5cf0045",
"doukutsu_rs": "6e40973b911b81ce799156cec69af0f716e8b0e8",
"duckstation": "7113d3c62a314cc6da7ef3bc12efec29f415ed6b",
"easyrpg": "d057e86ecc3482bc41dd88e5af75a59dea3360c6",
"ecwolf": "514cca6aa6df9f5552e9119d6173d3ff0d4e1a4a",
"eka2l1": "b6a0192caf919ef988bef44653153eb4815b3852",
"emucv": "1cf170a6eea802ed5f58d51482b06098f20ee7d6",
"emuscv": "37ca60ffca2affc6e2a264184cf16b64a5a9d1d1",
"emux_chip8": "76b0e06041b05ebbf723b101c04e0016729919cc",
"emux_gb": "ea951760e5ade5fb20dc4ca56f12e36bfd0a65fd",
"emux_nes": "043810ea5c9b3dcd4e82df1427c504331d3fc297",
"emux_sms": "8492133ac9cfdc6b765c506b9e549d83368b42da",
"ep128emu_core": "c6689eeb5b484e2e420073376f7956b01f4f9be0",
"eternal": "85267764b219d487ccb1ea77818bd535cde84911",
"fake08": "4e47454003da665a00094ed49e824aa902875a04",
"fbalpha2012": "593194b6f89ace127a52961a7a9c0cc251478d07",
"fbalpha2012_cps1": "1dd842a35fec91794f2ced1abc3fb5444c81c4cb",
"fbalpha2012_cps2": "676a6acae6f1c0f93b402b2cd22bfc7ef4132e12",
"fbalpha2012_cps3": "acf3699e74516c40ebb353ce7b73a78b2ba0c241",
"fbalpha2012_neogeo": "8002b6464d1ec6937ed57ba23191bb0b18272c3c",
"fbneo": "b34cff43b57cab0db469b2fa9c919ef43a48272c",
"fbneo_cps12": "b21e20af0b0d8d0c37945189a953b3a3791701ac",
"fbneo_neogeo": "c1ce3daa0c022a1c63eb3470a95f08c10eb2ae3d",
"fceumm": "84881bc01ed03bdcb2d47f89ab88c09296feb0d4",
"ffmpeg": "653c039661e9c564bce6b6325e5cc956296cf096",
"fixgb": "3335f3fee37e1843fc8060a7d58317a7fb61efef",
"fixnes": "1b1e191debbda3b45f51b20a2819a477fc7407cb",
"flycast": "47894ac43adc8876a630a69848e755cc34b7ad68",
"flycast_gles2": "7699704ac889ac17c6d0953b5518d3f0bfa4a6c5",
"fmsx": "2ba0cdc82c92a1625723e818b14fdf450ae532e1",
"freechaf": "43b80752c21ca2c26b610e5890a5d8edf08d8c45",
"freeintv": "40d3bc28c03ea0c83ebe8c907aa6f61d9547f063",
"freej2me": "690600ece3290683db35eef9c753f368e3ad6c2e",
"freej2me_plus": "1437a0bc7f9aa6fcb5a1e7b07aafe046eb3d76bd",
"frodo": "00b434e120ef8488ed82a30c01c488cc697ba091",
"fsuae": "ccbbfc85cedc9b7ab0f30702584d8b8114437f39",
"fuse": "2c53a6834974fe1d7c3e3467f57f3c7b9347eb8e",
"galaksija": "f27a232f8ae2ac12bae8436ed8fd220b745e8264",
"gam4980": "3d757498a3c9fa728c9df1ee18dd88faf73432fa",
"gambatte": "fc1e4e3b8f0ae047fab73ab516c5a0276370303d",
"gearboy": "670b1b90974ab6487b792fbeeddf5736afa4318f",
"gearcoleco": "2126bdabe8d1d8bb7d6dbcf3d1956502777bbc49",
"geargrafx": "79fe02a2840a276d1c834e31910ec91b3d9aa8b0",
"gearlynx": "624951f9e137ad27af7eabbf02550bc615a78dd2",
"gearsystem": "90d34a245aa2b6ac69a23a9af7952043f5d34425",
"genesis_plus_gx": "ae18384f5f4ba3e8417d430419d5fb4c1da02349",
"genesis_plus_gx_wide": "7a0deff1895e84f7475928814cdcdc9795471405",
"geolith": "e130b2796ea18dcce8cb8dcc6e35361d7c4c1682",
"gme": "285c70e52192a80a36b8bcb2f50538a8b8df3a5c",
"gong": "299c72ad78ab8c2498640c6eead875651e7072df",
"gpsp": "df4023b3909c82118427cac75699529f18904068",
"gw": "de42ed9ebe4ffe4f55d25e8523880f2c7c517ca0",
"handy": "e9f0c36320a31642202201bce0031f8a73c5a49e",
"hatari": "7641a27200948c97d046b7e0d559401894cc24ae",
"hatari2014": "54fde5cfffa09ec45d7a27a9b37978b7742677c2",
"hatarib": "9b2901327080b1c81406ba053dc1887fece953f3",
"hbmame": "54c043535e2635e936a8ac20912ab125394adf89",
"higan_sfc": "9bdcf2d0e92f0e9509b0bd91872c84c3562b9040",
"holani": "ec099eecd2d31c2c9355ca0f2b057c877d75e58a",
"imageviewer": "5eeb8e208aed6bec71427f6d6ac6bc9c93bb3365",
"irogb": "fd57733234d7219ff1758464406d5c67201b8c4e",
"ishiiruka": "d8c8c6c016451cb7c0d340eb51f150bf6f0983b3",
"jaxe": "5bb4cd5077edf0fce90a51116130894612460d2e",
"jollycv": "598b90e899a7d7d46f555c77c89c3a888d8eafb9",
"jumpnbump": "ed1a11061c6c53045f3b0d8c10128f719d5ea113",
"kronos": "e9711a1019be483c0d994ad1f3109e5ab06d0f2d",
"lowresnx": "93b3bc6c1d6204449a63b5c3d665c1cf230f0158",
"lutro": "7a5e1ba0d29c909b24f4da77296e0b794323fddd",
"m2000": "5a93b6679ed203a68db5cd8478a2fdfd03c86bf0",
"mame": "ae3a33aec0981c37c8caa63a7d8c86cbccbbc11c",
"mame2000": "326d7666692086df51e77f6ec89438872604ed18",
"mame2003": "f3ed8c023d555151e4d2cfce389ffdd2e50dab6e",
"mame2003_midway": "aa36dcb2e9295ff7d39a1c44fd641ab127c8d5d0",
"mame2003_plus": "07d5ae3f3857d574a19a67a4feb5e382bd419f79",
"mame2009": "1b673833aca392291e9643b24cda35539e2f7273",
"mame2010": "8f6b8d900859ce2f7f01bff1be5c60946076b829",
"mame2015": "8abad707dd5c0206eb6f0889adb9b46128ea0975",
"mame2016": "6231bcc43590a3adc36fa4c14ddefe4d373fa440",
"mamearcade": "86f27948d45ed51bf9c6d523ebc1d63d5d92e840",
"mamemess": "77add36a71f40fd7bfb2cd3ba5664520346805fe",
"mcsoftserve": "3afd94c19a09208f3871b1045dc72d552627d11c",
"mednafen_gba": "786e4c6b87cab0b1307d42eae3eb622b23af9e38",
"mednafen_lynx": "37bb123616c246a94947778a8657eca319681f0b",
"mednafen_ngp": "bed26ff8cd9af0808d002ad5cd5e7fd54f88cf68",
"mednafen_pce": "857686ae3294ca1a963631a666261f4b8d085f2b",
"mednafen_pce_fast": "5ad7408d4a1b695f5f1c3a45a84484d3217c758d",
"mednafen_pcfx": "3b8f3cbd6d4f08459b4678b9cadc1f8ddc6b4d9a",
"mednafen_psx": "be7cbe483f0aa3b5dffafec0834e6295e7c4b594",
"mednafen_psx_hw": "865be67008dbe11c2d42f61297ca1e73fb305383",
"mednafen_saturn": "ecc7bcf61bc670aacc964a70a7e7d60c5c931424",
"mednafen_snes": "388cb94ba0e16c6306abd092f82494c72e6c2696",
"mednafen_supafaust": "7eec8261629ac0ac438f1cf6a5109db993aa132f",
"mednafen_supergrafx": "decbd8567766a7b39d7ceed042e46bb2d5180916",
"mednafen_vb": "0d3399122790f3e72d25abaab995622474e720e2",
"mednafen_wswan": "24cf8e85a0d027d464afd71612bbdc2a1aa86c65",
"melonds": "ba1e3762404c290082ca202183704e3c711bb49c",
"melondsds": "82c511d23dfa7b8a3f3bb43b38c6598cd835740b",
"mesen": "82871f69374e2e9a4e8ac52009bed1a9c26a3c94",
"mesen-s": "e9d19eb873e5d1dc8c65f409c7603413f5c99a32",
"mesen2": "cb556adf4cca53da9d7bad80fc63cc92fb37bf34",
"mess2015": "b9260d933fae4b78c34c45ed1b3e6f12cddae7d5",
"meteor": "871d3b256cff96544ee8866e860c5f5afb293ef1",
"mgba": "90e1425af4bf9b1cfcef189d51b596e19c793ffd",
"minivmac": "224fc445e6ffaa2b1ec2805fd5a8bf54daadc273",
"mkxp-z": "a5a54cb7c2001afe6fcdd9a782584a3ee8f020df",
"mojozork": "131fd6fe964349afd49b77a3d1b7e3712ad7aca9",
"moonlight": "c85e0fd4e9a9296840b3a8005b94fd80a289ba37",
"mpv": "a611b24c7b920d70d3512cc9080d1edeec92f31e",
"mrboom": "00144a62ac4606bf3fb5c8f81a01459b25a57471",
"mu": "399f56a82a4810c9779b788771d349b4853e9d42",
"mupen64plus_next": "e1fb9671b409bb51eda8a149862629a0b9b3dc35",
"mupen64plus_next_develop": "3e39838b6f2893f515ea5ffbea13c6c481400cd3",
"mupen64plus_next_gles2": "7244f0c0eae4b58fa4614f6e821bd1ff76d9ae8c",
"mupen64plus_next_gles3": "888e3863516ae1ad7795b3c8615aa635551dd7fc",
"native32emu": "c8b84e368952bea94f0710fbf9f6b5559bd6b5ba",
"nekop2": "53b91335b9daf398a84757a2b777c5de87cac906",
"neocd": "0aae4e07e8ed9b02fbe8a60abe29c1e3484171c1",
"nes": "0fbb75c08c902ca7b92d43504caf428235861138",
"nestopia": "26fd40b1b9aa3403df5cf6761e4f3759235d2c55",
"nicaiemu": "73472736288c7fb9ca98a711faabdfaa4f28153b",
"noods": "14e06a6c4a191fbfaf34b1770bff9ed6588c5ada",
"np2kai": "c23d1e3b202102da7102e5f693a309079102d64b",
"nside_sfc_balanced": "999c0e4741eef4b2791b8415f7d8cd39de926c33",
"nuance": "da298e6f6f087ba3121489556970118ea6056522",
"numero": "ccdad6bca3a9f6698372db09cadbf36fe4eb4ed1",
"nxengine": "e49b16e3ec843757de41d32a070f0b89fd15cdab",
"o2em": "46784954fb8b851ea6a6a6257200682d62686947",
"oberon": "ceabfa3830324b751b965be51e527bcd2bbeb471",
"omicron": "6737820370170e5602f1b932bb9ffdc7fbca86ed",
"onscripter": "96105396ccb990a5d0ca2d7fcbe832701a58f3f2",
"onsyuri": "45f2acc5c2060ad23904a0bc2ccda43918391f3e",
"openlara": "7c4a8cacd891a65038c46da00e110d951de95f71",
"opentyrian": "5f3bae87cddae1c5cb5f51cef3ea43e3a45c0a2b",
"opera": "d8447c579efd7425edc06b6e03697abfeebee062",
"panda3ds": "f95daaa5e6e68dabe71e825655f09a7e3c7b17fd",
"parallel_n64": "320b88d1032d3caa5ae3b39469dacc005ccae59b",
"parallel_n64_debug": "96611a7235f47b06075d237ef8e5d33207d195a2",
"pascal_pong": "a1810d465843e18701c208c9ec2a92d783195fe3",
"pcee2": "d2d27eff25a2135155dce50aa4cd75255887aec1",
"pcem": "66a67eb4428d34d796fa1d2f6a6546ed8d4fa26a",
"pcsx1": "0726d1eebfb99cdf672c1424d1a8a0690bf6dfe2",
"pcsx2": "a00056b019ba696ee75fcd11d4fec0a9085c4b89",
"pcsx_rearmed": "33df7832b10c5253967ca18c63149d6c7dfc743b",
"pcsx_rearmed_interpreter": "14cc4bb82765c2c5c96074d5b117922178590499",
"pcsx_rearmed_neon": "6d374259a34c88735a502d1d2de18e7fc95e990f",
"pd777": "d7ab7e6e9c9755a62d4cde072fc4b8460da64594",
"picodrive": "9e539d30e23a7ee469a6780500fb1400de00c976",
"piece": "0fbc5df982831b9ac3b2e327c41f4d68d42c3620",
"play": "b91e6d04d33a86694d1e14ed8f3cddd962805f33",
"pocketcdg": "9a7a9606dfc265ced4e6b8d0356b245bff076fc2",
"pokemini": "07cabab917a3b8040d80538b843de778f006820e",
"pokketstation": "90f40d6c9c3dc0dd347db21afa07d47c1407f032",
"potator": "8a05b453d812a73f016b868340e4463890d60b95",
"ppsspp": "5037b12f2cb253a439532c99e30f8df36afe394c",
"prboom": "0566bbfddd39bd187fbbacdc82bedc2e2744e471",
"prosystem": "040018e58b3fbbff26aaa45476212ffdc4e13668",
"puae": "59fa936dff0fe1329c6f71925bddadb99b15c9fa",
"puae2021": "97c86e42fe956830e1a3e4539ed4b2ada04f6908",
"puzzlescript": "f53511e49d7e193c99de1293893f2c29c178400c",
"px68k": "9421920a2234919334a160ae57548adac0c83065",
"qemu": "7928c363b7a7d5a7c865d5daac7d088980be3ab7",
"quasi88": "431e0b3da4b4930b7eb16d139202934357867cf8",
"quicknes": "8fcfcf350a9122d30c51c096f3fd13457fa3b517",
"race": "048634385705b2bf33b299abd2c43a7db45a0abe",
"radio": "878d659f05691ceaddeb7bf60054cb8018416ba7",
"redbook": "e095305bd14d068068d93db1d056e01dd72e946b",
"reminiscence": "535c6a79c5639d80039eb5519cfa37a08b81e88b",
"remotejoy": "fc0d8dfd3d4a21e675cdde0de25e2b6db962aebd",
"retro8": "87cc2d79300807ba11f259e27f9101133f8079d6",
"retrodream": "0544849053aa85007b3a4a4162501fa8ad259d9a",
"romcleaner": "6c9181472df165dab7449100f01cf6def1d9d707",
"rp6502": "128440e0c79671b0bdd13bad6b04f05ccd12492f",
"rustation": "01fdab0903e30a19a0ca299fc15b8aa241843417",
"rustynes": "c231a5836ff48970ada68f39cb002ebba190dbb8",
"rvvm": "5fe5334df5112ff14bee0009f2de55656fb815a4",
"same_cdi": "2bc8421aef357ccb70d320a113045b1bc2f8aff1",
"sameboy": "900d258764ec864fdae7b33d26ccb1f261023193",
"sameduck": "7625ddea334c4f1747b10e3ab92459ee2a3b05d7",
"scummvm": "17d120c360629463b535c6c872ec7d4faf94fd1c",
"sdlpal": "66b1f82a9de6ebc8d924ff82d4b37951d6fcc5f9",
"simcp": "7bf05971cd0545aa22e9145547d30c2cbda19b08",
"skyemu": "1d82dca4afeab74bed4be85445b1538b9bf28a70",
"smsplus": "ea86bf40b6ce50c84dd9dc0905972f1c2a6e2222",
"snes9x": "aad2e41f7a6f9663bf93e080f9b75068c204dcb3",
"snes9x2002": "1535b62e6098a7c8f3890f2687bae215a4268900",
"snes9x2005": "93ec3600e80213fcaf26168c8837d264c6a64267",
"snes9x2005_plus": "b7436a7e01fa83ddd57c8c5d7fa573b5a45af597",
"snes9x2010": "a6eecb552435a0dc6a54883f7bb41263268111fc",
"spmp8000emu": "a7f2dd6a74ef2c3c511e7127ebc5c3e95c9a130a",
"squirreljme": "7f276aa61baab33b72452f6a693168be5ecf1285",
"stella": "5cdf4139790033096241f62d56cb2c53da32715e",
"stella2014": "508af340a5913b19b0e645d81bef2870364db91c",
"stella2023": "abf54de5705fdb4df7d12f7db7952edb63c24e27",
"stonesoup": "b624f68bc05611fac7e646b63dbe335fc83062ec",
"superbroswar": "8a4f806684cba1237a398e3a65ce297656aed642",
"supermodel": "1aafc1cc70d192108835666be46a7e1eb8974352",
"swanstation": "ab8a9d1a14ae6d7ce33a38ab0294b8ac55018745",
"syobonaction": "4c632b15e8931e8c3cea5b760e971f67a33cac57",
"tamalibretro": "acb163b7c45456fd0e880bd32ce9d75690299000",
"tempgba": "3cdeb907e9a14694cf74b4611395d9dd478cb3b5",
"test": "f85524c6b3b569b5acc16980e3df5246b6748043",
"test_netplay": "2e7efd155452a5d3c45fc947b0d8eddcba4a1b0b",
"testaudio_callback": "10be60661a53052d41bef0316b302b6439b8ab90",
"testaudio_no_callback": "8e4b54a4570c6c960200235391854b608234596f",
"testaudio_playback_wav": "42d6ccc92c644a79f25ac1c09c6fdaaffa382aef",
"testgl": "1d10d7a086c26abb669b6cc3d2d3a19334ddda02",
"testgl_compute_shaders": "2b90f8841c367ab9c38bb841ea82942d30dd8cf9",
"testgl_ff": "59f2bb4c129aca7dc26a28e4fd0ce90b4a892371",
"testinput_buttontest": "eb08c91990fa1af920d357aa01a735f1e597757b",
"testretroluxury": "029dae162931a067d8340dfa62f8ae5fe48dc1ae",
"testsw": "f17e8892effda5f94b44cc3be49063279d4727bf",
"testsw_vram": "07258f7b36225cc23bec5c248d6de327fd38a7dc",
"testvulkan": "7b6e9a630201cc3303223f8f907e54546e75a93b",
"testvulkan_async_compute": "5bab2d38b89555ef0dd2659b6fcc57208f1c2f30",
"tgbdual": "db1f5af20a973dd2ad28ac3a4e235ee5073be46b",
"theodore": "7829b8f95b223baa5fde439a40105143636183f5",
"thepowdertoy": "e54f44c6c8bd8507932fa05714c1691cebbea9e1",
"tia": "e9604c40852f8a02b0ea3ed487810d0071ccc6b9",
"tic80": "4c07f9a66921648a01e492580e6415a241c4a277",
"trident": "d3ca42ffb218faf3e96c231f13abd4f42926ec7d",
"tyrquake": "2b492b43b6bdcce3f62b7ccb06f81b68e5802e0d",
"uae4arm": "4609b8172db67dbc9b2c5a6e9c10287f4b07e867",
"ume2015": "cc668cd200e31492ac09f6b7f3e77e338f8f10aa",
"uw8": "c920638e44627f5ea9fb5a66cd05b925d3fa7744",
"uxn": "ae7f7069dd14125875d1474c97eba70c916e3b99",
"uzem": "e3cec8c8a0ccee9fba3884a80c8a3c909d513355",
"vaporspec": "390f79a6b346d6aa32afb5f5e6356b079a05578b",
"vba_next": "edbdb88f64a85755f2085f2cb80d232429f4848b",
"vbam": "a6a4751eb8e9048595a161ffeb78106334c9b18b",
"vecx": "c1126410f4d0d9abbb8b2b879eed9195613b19e3",
"vemulator": "4a16b56827e312a454c488129a96a55760956a8c",
"vice_x128": "ed36f29cea1b2f078b5a395e8ea3c8385f8c4156",
"vice_x64": "5f6626bd3a5f4960d6cc5b3ad7639e62ff3f2065",
"vice_x64dtv": "f51980a6b237b549d69bec751c4e3abe248b006f",
"vice_x64sc": "09c7aea1cb6740ba7e57993bd0fcf27f99a8eaf3",
"vice_xcbm2": "c8eb7f84bf9e377c75831b903990b8af18662391",
"vice_xcbm5x0": "8a1e1a78ef2546c58d8545eff2d3f41a2ec5b27c",
"vice_xpet": "2251efb6f5b029e0112dd255e2d691046a903363",
"vice_xplus4": "4b46c2f951c2f402cf0ff8c56160b7bfb4460d54",
"vice_xscpu64": "4dcef239dd83be5a33ede64a2070adf9a211ef72",
"vice_xvic": "33f3689b881367a2400cdead1d7b542a311b4dc2",
"video_processor": "5bdc74ff749976a560a534af0d06277153a5c206",
"vidtest": "4d8fe9a3f3033d06b5ec8ac28327965dcf951f0b",
"vircon32": "66d770352a9a1da119e8f766df17b0bfcbf7d757",
"virtualjaguar": "fc434223e248d32c38e46fcd84eae40116d1bbb6",
"virtualxt": "0f078c4357c3f6196faea56951930491455b47b8",
"vitaquake2": "18ae56936e0857d726df6b1805bcaf39f649e1c8",
"vitaquake2-rogue": "5740238c8c868c1bd77278565deb4d0380bff1f9",
"vitaquake2-xatrix": "9457b624a2ef954d8a956a9dbb1cade98a4f8535",
"vitaquake2-zaero": "b6544d6d13255bdc9811277bf301bf5b764e1912",
"vitaquake3": "0fa5379c56f04a121e8026a3694fa45e01a50514",
"vitavoyager": "0e17542eeeb904e6bbfedb34db8d628b58b4c09b",
"wasm4": "448263d66739c91a3d53c992253b49742416871c",
"wqxemu": "75b6fd2d3726386ebf3e2839ab1facbe87df3f7a",
"x1": "81e99f41eb8a048defc206383fbabdf281e53b6f",
"x64sdl": "4171f40c8a70256a2c5cb70db6a2b8fa733d9b3d",
"xemu": "e6033c0002a3311d6f0ba2bd6c1f399324d65e96",
"xrick": "2bb032e72b1432fa2c428d186db4ce55dfec8e2f",
"yabasanshiro": "bc1d40f813591ab89fb3c69af143976b636a6d89",
"yabause": "3b4247a760bff675b05f8436ce0b648cf4ea9e21",
"yaps2": "809a88a9100d3021461aba03ffdae5d09a7036f9",
"ymir": "13f216fa1b836ac742e907d65fb5f5d462307d95"
}
}
+228 -1
View File
@@ -1,6 +1,6 @@
{
"source": "fbneo",
"imported_at": "2026-10-04",
"imported_at": "2026-10-21",
"dats": {
"FinalBurn Neo - Arcade Games": "1.0.0.03",
"FinalBurn Neo - Astrocade Home Computer Games": "1.0.0.03",
@@ -1326,6 +1326,41 @@
"FinalBurn Neo - Arcade Games"
]
},
{
"dat": "FinalBurn Neo - Arcade Games",
"name": "invaders.zip",
"set": "invaders",
"description": "Space Invaders / Space Invaders M",
"members": [
{
"name": "9316b-0854_m739e.e1",
"size": 2048,
"crc32": "14e538b0",
"sha1": ""
},
{
"name": "9316b-0855_m739f.f1",
"size": 2048,
"crc32": "0ccead96",
"sha1": ""
},
{
"name": "9316b-0856_m739g.g1",
"size": 2048,
"crc32": "6bfaca4a",
"sha1": ""
},
{
"name": "9316b-0869_m739h.h1",
"size": 2048,
"crc32": "734f5ad8",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - Arcade Games"
]
},
{
"dat": "FinalBurn Neo - Arcade Games",
"name": "irrmaze.zip",
@@ -3536,6 +3571,65 @@
"FinalBurn Neo - Arcade Games"
]
},
{
"dat": "FinalBurn Neo - Arcade Games",
"name": "omegrace.zip",
"set": "omegrace",
"description": "Omega Race (set 1)",
"members": [
{
"name": "dvgprom.bin",
"size": 256,
"crc32": "d481e958",
"sha1": ""
},
{
"name": "omega.e1",
"size": 2048,
"crc32": "1d0fdf3a",
"sha1": ""
},
{
"name": "omega.f1",
"size": 2048,
"crc32": "d44c0814",
"sha1": ""
},
{
"name": "omega.j7",
"size": 4096,
"crc32": "8e8d4b54",
"sha1": ""
},
{
"name": "omega.k7",
"size": 4096,
"crc32": "6d10f197",
"sha1": ""
},
{
"name": "omega.l7",
"size": 4096,
"crc32": "edcd7a7d",
"sha1": ""
},
{
"name": "omega.m7",
"size": 4096,
"crc32": "0424d46e",
"sha1": ""
},
{
"name": "sound.k5",
"size": 2048,
"crc32": "7d426017",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - Arcade Games"
]
},
{
"dat": "FinalBurn Neo - Arcade Games",
"name": "pgm.zip",
@@ -7703,6 +7797,47 @@
"FinalBurn Neo - ColecoVision Games"
]
},
{
"dat": "FinalBurn Neo - ColecoVision Games",
"name": "omegrace.zip",
"set": "omegrace",
"description": "Omega Race (USA, Euro)",
"members": [
{
"name": "Omega Race (USA, Euro)(1983)(Coleco).rom",
"size": 16384,
"crc32": "0a0511c7",
"sha1": ""
},
{
"name": "coleco.rom",
"size": 8192,
"crc32": "3aa93ef3",
"sha1": ""
},
{
"name": "colecoa.rom",
"size": 8192,
"crc32": "39bb16fc",
"sha1": ""
},
{
"name": "czz50.rom",
"size": 16384,
"crc32": "4999abc6",
"sha1": ""
},
{
"name": "svi603.rom",
"size": 8192,
"crc32": "19e91b82",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - ColecoVision Games"
]
},
{
"dat": "FinalBurn Neo - ColecoVision Games",
"name": "pv2000.zip",
@@ -8116,6 +8251,41 @@
"FinalBurn Neo - MSX 1 Games"
]
},
{
"dat": "FinalBurn Neo - MSX 1 Games",
"name": "skydiver.zip",
"set": "skydiver",
"description": "Sky Diver (Japan)",
"members": [
{
"name": "Sky Diver (Japan, EN)(1984)(Hudson Soft)[BLOAD'CAS-',R].cas",
"size": 11734,
"crc32": "93a40a2c",
"sha1": ""
},
{
"name": "kanji.rom",
"size": 262144,
"crc32": "1f6406fb",
"sha1": ""
},
{
"name": "msx.rom",
"size": 32768,
"crc32": "a317e6b4",
"sha1": ""
},
{
"name": "msxj.rom",
"size": 32768,
"crc32": "071135e0",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - MSX 1 Games"
]
},
{
"dat": "FinalBurn Neo - Megadrive Games",
"name": "bubsys.zip",
@@ -8150,6 +8320,40 @@
"FinalBurn Neo - Megadrive Games"
]
},
{
"dat": "FinalBurn Neo - Megadrive Games",
"name": "ggenie1.zip",
"set": "ggenie1",
"description": "Game Genie (Euro, USA)",
"members": [
{
"name": "sgr001+sgr002.bin",
"size": 32768,
"crc32": "5f293e4c",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - Megadrive Games"
]
},
{
"dat": "FinalBurn Neo - Megadrive Games",
"name": "warrior.zip",
"set": "warrior",
"description": "Warrior of Rome (USA)",
"members": [
{
"name": "warrior of rome (usa).bin",
"size": 1048576,
"crc32": "5be10c6a",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - Megadrive Games"
]
},
{
"dat": "FinalBurn Neo - NES Games",
"name": "galaga.zip",
@@ -8555,6 +8759,29 @@
"FinalBurn Neo - ZX Spectrum Games"
]
},
{
"dat": "FinalBurn Neo - ZX Spectrum Games",
"name": "invaders.zip",
"set": "invaders",
"description": "Invaders (16K)",
"members": [
{
"name": "Invaders 16K (1983)(DK'Tronics).tap",
"size": 6795,
"crc32": "a6263eb9",
"sha1": ""
},
{
"name": "spectrum.rom",
"size": 16384,
"crc32": "ddee531f",
"sha1": ""
}
],
"dats": [
"FinalBurn Neo - ZX Spectrum Games"
]
},
{
"dat": "FinalBurn Neo - ZX Spectrum Games",
"name": "mach3.zip",
+24005 -1
View File
File diff suppressed because it is too large. Load diff
+163 -24
View File
@@ -40,6 +40,7 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
@@ -56,6 +57,7 @@ from common import ( # noqa: E402
list_registered_platforms,
load_emulator_profiles,
upstream_profile_index,
write_text_atomic,
yaml_load,
)
from exporter import discover_exporters # noqa: E402
@@ -65,6 +67,8 @@ from scripts.scraper.targets import discover_target_scrapers # noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[1]
CORE_INFO_REPO = "https://github.com/libretro/libretro-core-info"
# The git blob of every .info as last reviewed: `--record-coreinfo` writes it.
CORE_INFO_RECORD = REPO_ROOT / "platforms" / "_core_info.json"
NO_INTRO_MIRROR = "https://github.com/hugo19941994/auto-datfile-generator"
NO_INTRO_RELEASE = "Daily_Rebuild"
NO_INTRO_ASSET = "no-intro.zip"
@@ -107,6 +111,9 @@ class PlatformDiff:
files_changed: list[str] = field(default_factory=list)
cores_added: list[str] = field(default_factory=list)
cores_removed: list[str] = field(default_factory=list)
# Any other field the scrape writes: launchers, dual_mode_cores, a
# system's core, native_id or data_directories.
fields_changed: list[str] = field(default_factory=list)
@property
def content_changed(self) -> bool:
@@ -119,6 +126,7 @@ class PlatformDiff:
self.files_changed,
self.cores_added,
self.cores_removed,
self.fields_changed,
)
)
@@ -138,6 +146,7 @@ class PlatformDiff:
("~files", self.files_changed),
("+cores", self.cores_added),
("-cores", self.cores_removed),
("~fields", self.fields_changed),
):
if items:
shown = ", ".join(items[:SHOWN_ITEMS]) + (", ..." if len(items) > SHOWN_ITEMS else "")
@@ -177,6 +186,33 @@ def _keyed_files(files: list[dict]) -> dict[str, dict]:
return keyed
# Compared on their own above. `source` is the revision read: it moves with
# every upstream commit, and what that commit changes is diffed field by field.
_DIFFED_APART = frozenset({"version", "systems", "cores", "standalone_cores", "source"})
def _without_refs(value: object) -> object:
"""*value* without its `source_ref` lines.
A launcher cites lines of the revision read, so they shift with any
upstream commit above them (es_features.cfg:11638 read as :11649 one
RetroBat commit later). The cited content is what is compared.
"""
if isinstance(value, dict):
return {k: _without_refs(v) for k, v in value.items() if k != "source_ref"}
if isinstance(value, list):
return [_without_refs(v) for v in value]
return value
def _changed_fields(old: dict, new: dict, skip: frozenset[str], prefix: str = "") -> list[str]:
return [
f"{prefix}{key}"
for key in sorted(set(old) | set(new), key=str)
if key not in skip and _without_refs(old.get(key)) != _without_refs(new.get(key))
]
def diff_platform(old: dict, new: dict) -> PlatformDiff:
"""Compare two platform files the way a reviewer reads the diff.
@@ -205,6 +241,11 @@ def diff_platform(old: dict, new: dict) -> PlatformDiff:
for k in sorted(set(old_files) & set(new_files))
if old_files[k] != new_files[k]
)
diff.fields_changed.extend(_changed_fields(
old_systems[system] or {}, new_systems[system] or {},
frozenset({"files"}), f"{system}/",
))
diff.fields_changed = _changed_fields(old, new, _DIFFED_APART) + diff.fields_changed
old_cores, new_cores = _core_names(old), _core_names(new)
diff.cores_added = sorted(new_cores - old_cores)
diff.cores_removed = sorted(old_cores - new_cores)
@@ -432,6 +473,27 @@ def _transcribed_at(platforms_dir: Path, platform: str) -> float:
return newest
# How the truth model settles two cores' hashes: generate_truth reports it,
# and it says nothing about whether a cached original is current.
_MODEL_NOTES = ("WARNING: hash conflict",)
@contextlib.contextmanager
def _without_model_notes() -> Iterator[None]:
"""Keep the model build's own notes out of the freshness report.
Anything else written to stderr passes through.
"""
sink = io.StringIO()
try:
with contextlib.redirect_stderr(sink):
yield
finally:
for line in sink.getvalue().splitlines(keepends=True):
if not line.startswith(_MODEL_NOTES):
sys.stderr.write(line)
def check_native(
platforms_dir: Path, cache_dir: Path, truth_dir: Path,
published: Path = Path("exports"),
@@ -444,24 +506,25 @@ def check_native(
exporter_class = exporters.get(name)
if not exporter_class:
continue
try:
truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir))
except export_native.StaleTruth as exc:
findings.append(Finding("native", name, STALE, detail=str(exc)))
continue
systems, _ = build_native_model(truth or {}, scraped)
wanted = export_native.wanted_sources(exporter_class(), systems, scraped)
reason = native_cache_state(
wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
)
if reason:
reason += ", export_native.py --refresh-cache fetches it"
else:
reason = published_export_state(
name, exporter_class, truth_dir, platforms_dir, cache_dir, published
with _without_model_notes():
try:
truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir))
except export_native.StaleTruth as exc:
findings.append(Finding("native", name, STALE, detail=str(exc)))
continue
systems, _ = build_native_model(truth or {}, scraped)
wanted = export_native.wanted_sources(exporter_class(), systems, scraped)
reason = native_cache_state(
wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
)
if reason:
reason += f", export_native.py --platform {name} writes it again"
reason += ", export_native.py --refresh-cache fetches it"
else:
reason = published_export_state(
name, exporter_class, truth_dir, platforms_dir, cache_dir, published
)
if reason:
reason += f", export_native.py --platform {name} writes it again"
findings.append(
Finding(
"native",
@@ -660,8 +723,9 @@ def _age_days(stamp: str, now: datetime) -> int | None:
# --- core-info --------------------------------------------------------------
def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
"""Every core the libretro core-info repository describes."""
def core_info_blobs(cache_dir: str, offline: bool) -> dict[str, str] | None:
"""Every core the libretro core-info repository describes, with the git
blob of its .info."""
repo = upstream.parse_repo(CORE_INFO_REPO)
if repo is None:
return None
@@ -673,12 +737,54 @@ def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
)
if not isinstance(payload, list):
return None
names = []
blobs = {}
for entry in payload:
name = entry.get("name", "") if isinstance(entry, dict) else ""
if name.endswith("_libretro.info"):
names.append(name[: -len("_libretro.info")])
return sorted(names)
blobs[name[: -len("_libretro.info")]] = str(entry.get("sha") or "")
return dict(sorted(blobs.items()))
def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
"""Every core the libretro core-info repository describes."""
blobs = core_info_blobs(cache_dir, offline)
return None if blobs is None else list(blobs)
def load_core_info_record(path: Path = CORE_INFO_RECORD) -> dict[str, str] | None:
if not path.is_file():
return None
return dict((json.loads(path.read_text(encoding="utf-8")) or {}).get("blobs") or {})
def record_core_info(
blobs: dict[str, str], names: list[str], path: Path = CORE_INFO_RECORD
) -> list[str]:
"""Record the current blob of *names* (every .info when empty) as reviewed.
Naming the cores keeps a change nobody read out of the record.
"""
unknown = sorted(set(names) - set(blobs))
if unknown:
raise ValueError(f"no such .info in core-info: {', '.join(unknown)}")
recorded = load_core_info_record(path) or {}
recorded.update({n: blobs[n] for n in (names or blobs)})
payload = {"repository": CORE_INFO_REPO, "blobs": dict(sorted(recorded.items()))}
write_text_atomic(str(path), json.dumps(payload, indent=2) + "\n")
return sorted(names or blobs)
def coreinfo_drift(
blobs: dict[str, str], recorded: dict[str, str], profiles: dict[str, dict]
) -> list[str]:
"""Profiled cores whose .info is not the one recorded as reviewed.
The .info is no authority, but a new firmwareN_path in it is the
cheapest sign that a profile needs reading again.
"""
index = {key.casefold() for key in profile_name_index(profiles)}
return [n for n, sha in blobs.items()
if n.casefold() in index and recorded.get(n) != sha]
def coreinfo_gaps(
@@ -708,10 +814,14 @@ def coreinfo_gaps(
return unprofiled, standalone
def check_coreinfo(profiles: dict[str, dict], cache_dir: str, offline: bool) -> list[Finding]:
names = core_info_names(cache_dir, offline)
if names is None:
def check_coreinfo(
profiles: dict[str, dict], cache_dir: str, offline: bool,
record_path: Path = CORE_INFO_RECORD,
) -> list[Finding]:
blobs = core_info_blobs(cache_dir, offline)
if blobs is None:
return [Finding("coreinfo", "libretro-core-info", UNKNOWN, detail="listing unavailable")]
names = list(blobs)
unprofiled, standalone = coreinfo_gaps(names, profiles)
findings = [
Finding(
@@ -736,6 +846,22 @@ def check_coreinfo(profiles: dict[str, dict], cache_dir: str, offline: bool) ->
)
for name, key in standalone
)
recorded = load_core_info_record(record_path)
if recorded is None:
findings.append(Finding(
"coreinfo", "reviewed .info", UNKNOWN,
detail=f"no {record_path.name}: run check_freshness.py --record-coreinfo",
))
return findings
findings.extend(
Finding(
"coreinfo", name, STALE,
local=(recorded.get(name) or "not recorded")[:12], remote=blobs[name][:12],
detail=(".info changed since reviewed: read its firmware, then "
f"check_freshness.py --record-coreinfo {name}"),
)
for name in coreinfo_drift(blobs, recorded, profiles)
)
return findings
@@ -1278,8 +1404,21 @@ def main() -> int:
"--emulators-dir", default="emulators",
help="emulator profile directory (default: %(default)s)",
)
parser.add_argument(
"--record-coreinfo", nargs="*", metavar="CORE",
help="record the current .info of CORE (every core when none is named) "
f"as reviewed in {CORE_INFO_RECORD.relative_to(REPO_ROOT)}, then exit",
)
args = parser.parse_args()
if args.record_coreinfo is not None:
blobs = core_info_blobs(args.cache_dir, args.offline)
if not blobs:
raise RuntimeError("core-info listing unavailable: nothing recorded")
done = record_core_info(blobs, args.record_coreinfo)
print(f"recorded {len(done)} .info file(s) as reviewed")
return 0
areas = tuple(a.strip() for a in args.only.split(",")) if args.only else AREAS
unknown = sorted(set(areas) - set(AREAS))
if unknown:
+37 -35
View File
@@ -3225,46 +3225,48 @@ def _main(held: contextlib.ExitStack) -> None:
return
data_registry = load_data_dir_registry(args.platforms_dir)
if data_registry and not args.offline and not args.no_refresh_data:
from refresh_data_dirs import load_registry, refresh_all
# The output directory is claimed before the data directory refresh: a
# refresh waits for every run that holds a cache, so a second build
# during a pipeline sat silent for hours instead of being refused.
with _pack_output_lock(args.output_dir):
if data_registry and not args.offline and not args.no_refresh_data:
from refresh_data_dirs import load_registry, refresh_all
registry = load_registry(os.path.join(args.platforms_dir, "_data_dirs.yml"))
results = refresh_all(registry, force=args.refresh_data)
updated = sum(1 for v in results.values() if v)
if updated:
print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}")
failed = sorted(k for k, v in results.items() if v is None)
if failed:
print(f"WARNING: data directory refresh failed: {', '.join(failed)}")
if data_registry:
hold_data_dirs(held, data_registry)
registry = load_registry(os.path.join(args.platforms_dir, "_data_dirs.yml"))
results = refresh_all(registry, force=args.refresh_data)
updated = sum(1 for v in results.values() if v)
if updated:
print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}")
failed = sorted(k for k, v in results.items() if v is None)
if failed:
print(f"WARNING: data directory refresh failed: {', '.join(failed)}")
if data_registry:
hold_data_dirs(held, data_registry)
emu_profiles = load_emulator_profiles(args.emulators_dir)
emu_profiles = load_emulator_profiles(args.emulators_dir)
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms,
args.target,
args.platforms_dir,
is_all=args.all,
)
except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms,
args.target,
args.platforms_dir,
is_all=args.all,
)
except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
groups = group_identical_platforms(
platforms, args.platforms_dir, target_cores_cache if args.target else None
)
groups = group_identical_platforms(
platforms, args.platforms_dir, target_cores_cache if args.target else None
)
if args.manifest:
with _pack_output_lock(args.output_dir):
if args.manifest:
_run_manifest_mode(
args, groups, db, zip_contents, emu_profiles, target_cores_cache
)
else:
with _pack_output_lock(args.output_dir):
else:
_run_platform_packs(
args,
groups,
@@ -3275,9 +3277,9 @@ def _main(held: contextlib.ExitStack) -> None:
target_cores_cache,
system_filter,
)
if args.verify_packs:
with _pack_output_lock(args.output_dir, exclusive=False):
_run_verify_packs(args)
if not args.manifest and args.verify_packs:
with _pack_output_lock(args.output_dir, exclusive=False):
_run_verify_packs(args)
# Manifest generation (JSON inventory for install.py)
+34 -13
View File
@@ -164,18 +164,15 @@ def _fetch_asset(
if os.path.isdir(cached):
return None
if os.path.exists(cached) and not (expected_sha1 or expected_md5):
return cached
if os.path.exists(cached):
try:
hashes = compute_hashes(cached) if (expected_sha1 or expected_md5) else {}
except FileNotFoundError:
hashes = None
if hashes is None:
snapshot = _read_once(cached)
if snapshot is None:
pass
elif not (expected_sha1 or expected_md5) or _matches(
hashes, expected_sha1, expected_md5
):
elif _matches(snapshot[0], expected_sha1, expected_md5):
return cached
elif not _replaced_on_release(name, cached, offline):
elif not _replaced_on_release(name, snapshot[0]["sha256"], snapshot[1], offline):
# A copy of this asset that answers another hash: the caller
# wants a different revision under the same name, and the
# release still serves this one. Keeping it is what lets the
@@ -249,8 +246,32 @@ def _asset_urls(name: str) -> list[tuple[str, str]]:
]
def _replaced_on_release(name: str, cached: str, offline: bool) -> bool:
"""Whether the release now serves other bytes than the cached copy.
def _read_once(path: str) -> tuple[dict[str, str], int] | None:
"""Digests and size of the file at *path*, all read from one file, or
None when it is gone.
Every judgement on a cache copy reads this one snapshot. Hashed, then
hashed again by path for the release check, a copy another run promoted
in between answered the second question: the copy the caller wanted was
in the cache and the caller was told there was none.
"""
# A promotion is an os.replace, so it changes the inode: the same inode
# on both sides of the read means the digests describe that one file.
for _attempt in range(3):
try:
before = os.stat(path)
hashes = compute_hashes(path)
after = os.stat(path)
except FileNotFoundError:
return None
if (before.st_ino, before.st_mtime_ns) == (after.st_ino, after.st_mtime_ns):
return hashes, after.st_size
return None
def _replaced_on_release(name: str, sha256: str, size: int, offline: bool) -> bool:
"""Whether the release now serves other bytes than the cached copy,
whose *sha256* and *size* the caller read.
The release lists a sha256 per asset: a dump rebuilt at the same size
(the eight 251658304-byte DSi NANDs) differs only there, and judged by
@@ -266,9 +287,9 @@ def _replaced_on_release(name: str, cached: str, offline: bool) -> bool:
None,
)
if digest:
return compute_hashes(cached, frozenset({"sha256"}))["sha256"] != digest
return sha256 != digest
served = _served_size(name)
return served is not None and served != os.path.getsize(cached)
return served is not None and served != size
_digests_cache: dict[str, str] | None = None
+56 -8
View File
@@ -23,6 +23,7 @@ import tempfile
import urllib.error
import urllib.request
import zipfile
from collections.abc import Iterator
from pathlib import Path
from artifacts import file_lock as _file_lock
from common import github_token, platform_arg, require_platform, yaml_load
@@ -61,7 +62,34 @@ def cache_lock(
swapped the tree shipped part of it, or nothing.
"""
cache_dir = Path(cache_dir)
return _file_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared=shared)
return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
@contextlib.contextmanager
def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
"""file_lock, saying so when it has to wait.
A pipeline holds every cache for its whole run: a refresh started beside
it waited hours with nothing on screen.
"""
try:
import fcntl
except ImportError:
with _file_lock(lock_path, shared=shared):
yield
return
lock_path.parent.mkdir(parents=True, exist_ok=True)
mode = fcntl.LOCK_SH if shared else fcntl.LOCK_EX
with open(lock_path, "a") as handle:
try:
fcntl.flock(handle, mode | fcntl.LOCK_NB)
except BlockingIOError:
log.warning("waiting for %s: another run holds it", cache_dir)
fcntl.flock(handle, mode)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
def hold_data_dirs(stack: contextlib.ExitStack, registry: dict) -> None:
@@ -383,6 +411,14 @@ def _get_remote_etag(source_url: str) -> str | None:
return None
def _short_tag(tag: str | None) -> str:
"""A commit sha or ETag cut for a log line, quotes dropped first: a
quoted ETag cut at 12 characters kept its opening quote only."""
if not tag:
return "none"
return tag.strip().removeprefix("W/").strip('"')[:12] or "none"
def refresh_entry(
key: str,
entry: dict,
@@ -431,16 +467,17 @@ def _refresh_entry(
needs_refresh = remote_tag != cached_tag
if not needs_refresh:
log.info("[%s] up to date (tag: %s)", key, (cached_tag or "?")[:12])
log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
return False
if dry_run:
log.info(
"[%s] would refresh (type: %s, cached: %s)",
key,
source_type,
cached_tag or "none",
)
if remote_tag is not None:
reason = f"cached: {_short_tag(cached_tag)}, remote: {_short_tag(remote_tag)}"
elif force:
reason = "forced"
else:
reason = "not cached"
log.info("[%s] would refresh (type: %s, %s)", key, source_type, reason)
return True
if remote_tag is None:
@@ -516,6 +553,16 @@ def refresh_all(
return results
def summarize(outcomes: dict[str, bool | None], dry_run: bool) -> str:
"""One closing line: how many entries changed, were current, failed."""
refreshed = sum(1 for o in outcomes.values() if o is True)
current = sum(1 for o in outcomes.values() if o is False)
failed = sum(1 for o in outcomes.values() if o is None)
verb = "would refresh" if dry_run else "refreshed"
line = f"{len(outcomes)} entries: {refreshed} {verb}, {current} up to date"
return f"{line}, {failed} failed" if failed else line
def main() -> None:
parser = argparse.ArgumentParser(
allow_abbrev=False,
@@ -567,6 +614,7 @@ def main() -> None:
registry, force=args.force, dry_run=args.dry_run, platform=args.platform
)
log.info(summarize(outcomes, args.dry_run))
failed = sorted(key for key, outcome in outcomes.items() if outcome is None)
if failed:
log.error("refresh failed: %s", ", ".join(failed))
+16 -3
View File
@@ -25,7 +25,8 @@ Mapping (stale -> command):
catalogs/fbneo recipes python -m scripts.scraper.romset_dat_importer
--source fbneo --fetch
Surfaced only (no refresh): coreinfo gaps, redump/no-intro/tosec packs,
Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
(the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
a human read (new .info file, manual DAT download, PyPI version bump).
@@ -46,12 +47,14 @@ import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
LOG_DIR = REPO_ROOT / "tmp" / "refresh_stale"
CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
@@ -91,6 +94,15 @@ def _load_platform_registry() -> dict[str, dict]:
return (yaml.safe_load(fh) or {}).get("platforms") or {}
@lru_cache(maxsize=1)
def _data_dir_keys() -> frozenset[str]:
"""The entries refresh_data_dirs.py --key accepts."""
import yaml
with DATA_DIRS_REGISTRY.open(encoding="utf-8") as fh:
return frozenset((yaml.safe_load(fh) or {}).get("data_directories") or {})
def _run_check_freshness(areas: tuple[str, ...], extra: list[str]) -> list[dict]:
"""Run check_freshness.py --json and return its findings."""
cmd = [sys.executable, str(CHECK_FRESHNESS), "--json"]
@@ -246,8 +258,9 @@ def _command_for(
]
if area == "data":
if subject == "_data_dirs.yml":
# The registry file itself cannot be refreshed, but its entries can.
if subject not in _data_dir_keys():
# "_data_dirs.yml" and "buildbot system" name the registry and
# what it does not track yet: a human adds the entry.
return None
return [
sys.executable,
+17
View File
@@ -163,6 +163,11 @@ def fetch_bytes(
class BaseScraper(ABC):
"""Abstract base class for platform BIOS requirement scrapers."""
# False for a platform whose file list is inherited and whose scraper
# reads only its core list (RetroPie). Otherwise an empty read is an
# upstream that changed shape, never a platform with no file.
lists_files: bool = True
def __init__(self, url: str = "") -> None:
self.url = url
self._raw_data: str | None = None
@@ -367,6 +372,12 @@ def scraper_cli(
reqs = scraper.fetch_requirements()
except (OSError, ValueError, RuntimeError) as e:
_exit_on_scrape_error(e)
if scraper.lists_files and not reqs:
# Written, an empty read became a platform with no system that
# verify and the pack count 0/0 and call consistent.
_exit_on_scrape_error(RuntimeError(
f"{scraper_class.__module__}: upstream returned no BIOS entry"
))
if args.dry_run:
by_system: dict[str, list] = {}
@@ -426,6 +437,12 @@ def scraper_cli(
if req.zipped_file:
entry["zipped_file"] = req.zipped_file
config["systems"][sys_id]["files"].append(entry)
if scraper.lists_files and not any(
(system or {}).get("files") for system in (config.get("systems") or {}).values()
):
_exit_on_scrape_error(RuntimeError(
f"{scraper_class.__module__}: the generated platform lists no file"
))
# Merge into existing YAML: preserve fields the scraper doesn't generate
# (data_directories, case_insensitive_fs, manually added metadata).
# The scraper replaces systems + files, everything else is preserved.
+6
View File
@@ -42,6 +42,8 @@ def _is_cache_fresh() -> bool:
return False
try:
data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
if not data.get("bios_sets"):
return False
fetched_at = datetime.fromisoformat(data["fetched_at"])
return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
except (json.JSONDecodeError, KeyError, ValueError):
@@ -178,6 +180,10 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
log.info("parsing source tree")
bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
if not bios_sets or not commit:
# An empty read was cached for a day and merged nothing, and
# the pipeline step reported OK.
raise RuntimeError(f"no BIOS set read at FBNeo {version} ({commit[:12]})")
cache: dict[str, Any] = {
"source": "finalburnneo/FBNeo",
+6 -1
View File
@@ -62,7 +62,7 @@ def _load_cache() -> dict[str, Any] | None:
def _is_stale(cache: dict[str, Any] | None) -> bool:
if cache is None:
if cache is None or not cache.get("bios_sets"):
return True
fetched_at = cache.get("fetched_at")
if not fetched_at:
@@ -292,6 +292,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
bios_sets = parse_mame_source_tree(str(_CLONE_DIR))
version = _parse_version_tag(tag)
commit = _get_commit()
if not bios_sets or not commit:
# An empty read was cached for a day and every profile entry
# came out "out of scope", so nothing was refreshed and the
# pipeline step reported OK.
raise RuntimeError(f"no BIOS root set read at {tag}")
data: dict[str, Any] = {
"source": "mamedev/mame",
+4
View File
@@ -59,6 +59,8 @@ _PACKAGE_DIRS = _SCRIPT_SECTIONS
class Scraper(BaseScraper):
"""Scraper for the RetroPie package list."""
lists_files = False
def __init__(self, url: str | None = None) -> None:
# The package list is read from master: the commit read is recorded.
if url is None:
@@ -122,6 +124,8 @@ class Scraper(BaseScraper):
else:
found.add(match.group(1))
if not found:
raise ValueError(f"{self.url}: no rp_module_id under {SCRIPTMODULES}/")
self._modules = sorted(found)
return self._modules
+73 -16
View File
@@ -15,7 +15,9 @@ in its repository, so both are fetched without a browser:
Only sets the collection has a reason to know are kept, so the committed
snapshot stays small: the archive names that emulator profiles or platform
BIOS lists reference. ``--all`` keeps everything.
BIOS lists reference, and the sets holding a dump-catalog entry the
collection lacks (provenance_report names the set as where to find it).
``--all`` keeps everything.
Members without a CRC32 are undumped. A real romset does not carry them, so
they are dropped from the recipe rather than disqualifying the whole set.
@@ -42,7 +44,7 @@ from ..common import (
load_emulator_profiles,
load_platform_config,
)
from ..dumpcatalog import build_snapshot, write_snapshot
from ..dumpcatalog import build_snapshot, load_provenance_snapshots, write_snapshot
from .logiqx_parser import LogiqxDat, parse_logiqx
MAX_MEMBER_SIZE = 200 * 1024 * 1024
@@ -74,8 +76,45 @@ def relevant_archives(emulators_dir: str, platforms_dir: str) -> set[str]:
return names
Targets = dict[tuple[str, int], set[str]]
def catalog_targets(provenance_dir: str, db_path: str) -> Targets:
"""(crc32, size) -> sha1 of each dump-catalog entry the collection lacks.
fm7740sx carries DICROM.rom, EXTSUB.rom and KANJI2.rom, which No-Intro
lists and the collection misses: filtered on archive names alone, the
snapshot never held the set and the missing report named no source.
"""
with open(db_path, encoding="utf-8") as handle:
held = set(json.load(handle).get("files", {}))
targets: Targets = {}
for snapshot in load_provenance_snapshots(provenance_dir).values():
for entry in snapshot.get("entries", []):
crc, size = str(entry.get("crc32") or "").lower(), entry.get("size")
sha1 = str(entry.get("sha1") or "").lower()
if not crc or size is None or (sha1 and sha1 in held):
continue
targets.setdefault((crc.zfill(8), int(size)), set()).add(sha1)
return targets
def holds_target(members: list[dict], targets: Targets | None) -> bool:
"""Whether a set carries a missing catalog dump. A sha1 on both sides
must agree: crc32 and size alone collide across 40 000 ROMs."""
if not targets:
return False
for member in members:
key = (str(member.get("crc32") or "").lower().zfill(8), int(member.get("size") or 0))
wanted = targets.get(key)
sha1 = str(member.get("sha1") or "").lower()
if wanted is not None and (not sha1 or "" in wanted or sha1 in wanted):
return True
return False
def recipe_entries(
dat: LogiqxDat, keep: set[str] | None = None
dat: LogiqxDat, keep: set[str] | None = None, targets: Targets | None = None
) -> list[dict]:
"""Group a parsed DAT into one recipe per set."""
by_set: dict[str, list] = {}
@@ -87,7 +126,9 @@ def recipe_entries(
entries: list[dict] = []
for set_name, roms in sorted(by_set.items()):
archive = f"{set_name}.zip"
if keep is not None and archive not in keep:
if keep is not None and archive not in keep and not holds_target(
[{"crc32": r.crc32, "size": r.size, "sha1": r.sha1} for r in roms], targets
):
continue
entries.append(
{
@@ -149,9 +190,12 @@ def _machine_roms(blob: bytes) -> list[dict]:
def _scan_listxml(
stream: io.BufferedIOBase, selector: Callable[[str], bool]
stream: io.BufferedIOBase,
selector: Callable[[str], bool],
targets: Targets | None = None,
) -> dict[str, tuple[str | None, list[dict]]]:
"""Stream a MAME -listxml document, keeping the machines *selector* wants.
"""Stream a MAME -listxml document, keeping the machines *selector* wants
and those holding one of *targets*.
The document is 311 MB for MAME 0.289, so it is never held whole: the
reader keeps a sliding window just large enough to close the element it
@@ -170,19 +214,22 @@ def _scan_listxml(
if end == -1:
break
name = match.group(1).decode()
if selector(name):
parent = _ROMOF.search(match.group(2).decode())
found[name] = (
parent.group(1) if parent else None,
_machine_roms(buffer[match.end() : end]),
)
wanted = selector(name)
if wanted or targets:
roms = _machine_roms(buffer[match.end() : end])
if wanted or holds_target(roms, targets):
parent = _ROMOF.search(match.group(2).decode())
found[name] = (parent.group(1) if parent else None, roms)
position = end + len(b"</machine>")
buffer = buffer[position:] if position else buffer[-(2 * LISTXML_CHUNK) :]
return found
def listxml_entries(
open_stream: Callable[[], io.BufferedIOBase], label: str, keep: set[str] | None
open_stream: Callable[[], io.BufferedIOBase],
label: str,
keep: set[str] | None,
targets: Targets | None = None,
) -> list[dict]:
"""Recipes from a MAME -listxml document, parents resolved.
@@ -193,7 +240,7 @@ def listxml_entries(
"""
with open_stream() as stream:
wanted = _scan_listxml(
stream, lambda name: keep is None or f"{name}.zip" in keep
stream, lambda name: keep is None or f"{name}.zip" in keep, targets
)
requested = set(wanted)
parents = {
@@ -484,6 +531,15 @@ def main() -> int:
parser.add_argument(
"--all", action="store_true", help="keep every set, not only referenced ones"
)
parser.add_argument(
"--provenance-dir", default="provenance",
help="dump catalogs whose missing entries keep the sets holding them "
"(default: %(default)s)",
)
parser.add_argument(
"--db", default="database.json",
help="collection database the catalogs are compared with (default: %(default)s)",
)
parser.add_argument(
"--dry-run", action="store_true",
help="print what would change and write nothing",
@@ -511,6 +567,7 @@ def main() -> int:
if args.all
else relevant_archives(args.emulators_dir, args.platforms_dir)
)
targets = None if args.all else catalog_targets(args.provenance_dir, args.db)
dats: dict[str, str] = {}
entries: list[dict] = []
@@ -518,7 +575,7 @@ def main() -> int:
if _is_listxml(pack):
label = f"MAME {pack.stem.replace('lx', '')}"
entries = listxml_entries(_listxml_opener(pack), label, keep)
entries = listxml_entries(_listxml_opener(pack), label, keep, targets)
dats[label] = pack.stem
print(f" {label}: {len(entries)} sets retenus")
members = sum(len(entry["members"]) for entry in entries)
@@ -539,7 +596,7 @@ def main() -> int:
if not dat.roms:
skipped += 1
continue
found = recipe_entries(dat, keep)
found = recipe_entries(dat, keep, targets)
if not found:
continue
dats[dat.name or member] = dat.version
+1
View File
@@ -43,6 +43,7 @@ class BaseTargetScraper(ABC):
r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/",
r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/",
r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/",
r"^https://codeload\.github\.com/[^/]+/[^/]+/(?:tar\.gz|zip)/([^/]+)$",
):
match = re.match(pattern, source)
if match:
@@ -1,8 +1,11 @@
"""Scraper for EmuDeck emulator targets.
Sources:
SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh
Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1
SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh, read at the
revision platforms/emudeck.yml was scraped at
Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1, default branch:
the platform file is read from dragoonDorise/EmuDeck alone and
pins no emudeck-we revision
"""
from __future__ import annotations
@@ -89,8 +92,18 @@ def _script_to_core(filename: str) -> str | None:
class Scraper(BaseTargetScraper):
"""Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck") -> None:
def __init__(
self, url: str = "https://github.com/dragoonDorise/EmuDeck", revision: str | None = None
) -> None:
super().__init__(url=url)
self._revision = revision
@property
def revision(self) -> str:
"""The checkBIOS.sh commit platforms/emudeck.yml was read at."""
if self._revision is None:
self._revision = self.pinned_revision(PLATFORM_NAME)
return self._revision
def _fetch_cores_for_target(
self, api_url: str, label: str, arch: str = "x86_64"
@@ -141,7 +154,9 @@ class Scraper(BaseTargetScraper):
raise RuntimeError(f"no {arch} core list in {target_path}")
def fetch_targets(self) -> dict:
steamos_cores = self._fetch_cores_for_target(STEAMOS_API, "SteamOS")
steamos_cores = self._fetch_cores_for_target(
f"{STEAMOS_API}?ref={self.revision}", "SteamOS"
)
windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows")
targets: dict[str, dict] = {}
@@ -158,7 +173,7 @@ class Scraper(BaseTargetScraper):
return {
"platform": "emudeck",
"source": self.url,
"source": f"{self.url}/tree/{self.revision}",
"scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"targets": targets,
}
@@ -1,6 +1,7 @@
"""Scraper for RetroPie package availability per platform.
Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules
Source: https://github.com/RetroPie/RetroPie-Setup/tree/<rev>/scriptmodules,
read at the revision platforms/retropie.yml was scraped at.
Parses rp_module_id and rp_module_flags from each scriptmodule of the
package sections the platform list reads (emulators, libretrocores, ports)
to determine which platforms each package supports.
@@ -26,7 +27,7 @@ GITHUB_API_URL = (
"https://api.github.com/repos/RetroPie/RetroPie-Setup/contents/scriptmodules"
)
RAW_BASE_URL = (
"https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master/scriptmodules/"
"https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/{rev}/scriptmodules/"
)
# The sections the platform scraper reads: a standalone package missing here
# is filtered out of every targeted pack.
@@ -114,14 +115,23 @@ def _parse_module(content: str) -> tuple[str | None, str]:
class Scraper(BaseTargetScraper):
"""Fetches RetroPie package availability by parsing scriptmodules."""
def __init__(self, url: str = GITHUB_API_URL) -> None:
def __init__(self, url: str = GITHUB_API_URL, revision: str | None = None) -> None:
super().__init__(url=url)
self._revision = revision
@property
def revision(self) -> str:
"""The commit the package list was read at: the flags of another
revision filter packages the platform file does not describe."""
if self._revision is None:
self._revision = self.pinned_revision(PLATFORM_NAME)
return self._revision
def _list_scriptmodules(self) -> list[str]:
"""Return section/filename for every .sh of the package sections."""
names: list[str] = []
for section in PACKAGE_DIRS:
url = f"{self.url}/{section}"
url = f"{self.url}/{section}?ref={self.revision}"
entries = json.loads(_fetch(url, accept="application/vnd.github+json"))
listed = [
f"{section}/{e['name']}" for e in entries if e.get("name", "").endswith(".sh")
@@ -132,7 +142,7 @@ class Scraper(BaseTargetScraper):
return names
def _fetch_module(self, filename: str) -> str:
return _fetch(f"{RAW_BASE_URL}{filename}")
return _fetch(f"{RAW_BASE_URL.format(rev=self.revision)}{filename}")
def fetch_targets(self) -> dict:
print(" listing RetroPie scriptmodules...", file=sys.stderr)
@@ -170,7 +180,7 @@ class Scraper(BaseTargetScraper):
return {
"platform": "retropie",
"source": self.url,
"source": f"{self.url}?ref={self.revision}",
"scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"targets": targets,
}
+30
View File
@@ -110,6 +110,36 @@ class PackLockCliTest(unittest.TestCase):
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
self.assertIn("is in use by another run", proc.stdout)
def test_a_held_output_is_refused_before_the_data_refresh(self):
"""The refresh waits for every run holding a data tree, and a
pipeline holds them all for its whole run: probed after the refresh,
a second build sat silent for hours instead of being refused."""
from refresh_data_dirs import cache_lock # noqa: PLC0415
platforms = Path(self.dir) / "platforms"
platforms.mkdir()
for name in ("_registry.yml", "_shared.yml", "misterfpga.yml"):
shutil.copyfile(REPO_ROOT / "platforms" / name, platforms / name)
cache = Path(self.dir) / "cache" / "probe"
cache.mkdir(parents=True)
(platforms / "_data_dirs.yml").write_text(
"data_directories:\n probe:\n source_type: zip\n"
f" source_url: https://example.invalid/probe.zip\n"
f" local_cache: {cache}\n"
)
output = Path(self.dir) / "out"
with artifact_lock(str(output)), cache_lock(cache):
try:
proc = subprocess.run(
[sys.executable, "scripts/generate_pack.py", "--platform", "misterfpga",
"--platforms-dir", str(platforms), "--output-dir", str(output)],
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=120,
)
except subprocess.TimeoutExpired:
self.fail("generate_pack waited on the data tree instead of refusing")
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
self.assertIn("is in use by another run", proc.stdout)
def test_every_writing_mode_refuses_a_locked_output(self):
"""Two of seven modes took the lock, and five wrote straight into it.
+75
View File
@@ -87,6 +87,25 @@ class NativeCacheTests(unittest.TestCase):
self.assertEqual({r.status for r in rows}, {cf.STALE})
self.assertIn("batocera", {r.subject for r in rows})
def test_the_model_build_notes_stay_out_of_the_report(self):
"""generate_truth reports how two cores' hashes were settled. Under
`--only native` they opened the report and named no step to take."""
import contextlib
import io
def noisy(truth, scraped):
print("WARNING: hash conflict for GameIndex.yaml (sha1: a vs b, core yaps2)",
file=sys.stderr)
print("other diagnostic", file=sys.stderr)
return {}, None
err = io.StringIO()
with mock.patch.object(cf, "build_native_model", side_effect=noisy), \
contextlib.redirect_stderr(err):
cf.check_native(REPO_ROOT / "platforms", self.cache, self.cache / "truth")
self.assertNotIn("hash conflict", err.getvalue())
self.assertIn("other diagnostic", err.getvalue())
class PublishedExportTests(unittest.TestCase):
"""exports/ is committed and published, and CI cannot rebuild it."""
@@ -165,6 +184,32 @@ class PlatformDiffTests(unittest.TestCase):
self.assertEqual(diff.cores_removed, ["b"])
self.assertIn("+cores 2", diff.summary())
def test_every_other_scraped_field_is_a_change(self):
"""launchers feed verify, dual_mode_cores the grouping, a system's
data_directories the pack: a rescrape that moves only them is stale."""
old = self._platform()
old["launchers"] = [{"emulator": "e"}]
old["dual_mode_cores"] = ["flycast"]
old["systems"]["sys"].update(core="c1", native_id="psx",
data_directories=[{"ref": "x"}])
new = self._platform()
new["launchers"] = []
new["systems"]["sys"].update(core="c2", native_id="psx")
diff = cf.diff_platform(old, new)
self.assertTrue(diff.content_changed)
self.assertEqual(
diff.fields_changed,
["dual_mode_cores", "launchers", "sys/core", "sys/data_directories"],
)
self.assertIn("~fields 4", diff.summary())
def test_the_revision_read_alone_is_not_a_change(self):
old, new = self._platform(), self._platform()
old["source"], new["source"] = "https://x/tar.gz/aaa", "https://x/tar.gz/bbb"
old["launchers"] = [{"emulator": "e", "source_ref": "es_features.cfg:11638-11646"}]
new["launchers"] = [{"emulator": "e", "source_ref": "es_features.cfg:11649-11657"}]
self.assertFalse(cf.diff_platform(old, new).changed)
def test_target_stamp_is_not_a_change(self):
old = {"scraped_at": "2026-01-01", "targets": {"t": {"cores": ["a"]}}}
new = {"scraped_at": "2026-02-01", "targets": {"t": {"cores": ["a"]}}}
@@ -240,6 +285,36 @@ class CoreResolutionTests(unittest.TestCase):
self.assertEqual(unprofiled, ["wqxemu"])
self.assertEqual(standalone, [("eka2l1", "eka2l1")])
def test_a_changed_info_of_a_profiled_core_is_stale(self):
"""A firmwareN_path added to a profiled core's .info changes its blob:
the names alone said OK."""
blobs = {"mednafen_psx": "new", "eka2l1": "same", "wqxemu": "other"}
recorded = {"mednafen_psx": "old", "eka2l1": "same"}
with tempfile.TemporaryDirectory() as tmp:
record = Path(tmp) / "_core_info.json"
record.write_text(cf.json.dumps({"blobs": recorded}))
with mock.patch.object(cf, "core_info_blobs", return_value=blobs):
rows = cf.check_coreinfo(self.PROFILES, "", True, record)
stale = {r.subject: r for r in rows if r.status == cf.STALE}
self.assertIn("mednafen_psx", stale)
self.assertIn("--record-coreinfo mednafen_psx", stale["mednafen_psx"].detail)
self.assertNotIn(".info changed", stale["eka2l1"].detail)
self.assertEqual(cf.coreinfo_drift(blobs, recorded, self.PROFILES), ["mednafen_psx"])
def test_recording_a_core_leaves_the_others_unread(self):
with tempfile.TemporaryDirectory() as tmp:
record = Path(tmp) / "_core_info.json"
record.write_text(cf.json.dumps({"blobs": {"a": "1", "b": "1"}}))
cf.record_core_info({"a": "2", "b": "2"}, ["a"], record)
self.assertEqual(cf.load_core_info_record(record), {"a": "2", "b": "1"})
with self.assertRaises(ValueError):
cf.record_core_info({"a": "2"}, ["ghost"], record)
def test_the_committed_record_holds_git_blobs(self):
recorded = cf.load_core_info_record()
self.assertIsNotNone(recorded)
self.assertTrue(all(len(sha) == 40 for sha in recorded.values()))
class PinParsingTests(unittest.TestCase):
WORKFLOW = """
+2 -1
View File
@@ -356,7 +356,8 @@ class TheScraperClonesTheReleaseItLabels(unittest.TestCase):
with mock.patch.object(scraper, "_get_release_tag", return_value="mame0289"), \
mock.patch.object(scraper, "_run_git",
side_effect=lambda args, cwd=None: calls.append(args)), \
mock.patch.object(scraper, "parse_mame_source_tree", return_value={}), \
mock.patch.object(scraper, "parse_mame_source_tree",
return_value={"neogeo": {"roms": []}}), \
mock.patch.object(scraper, "_get_commit", return_value="c" * 40), \
mock.patch.object(scraper, "_write_cache"), \
mock.patch.object(scraper, "_cleanup"), \
+28 -1
View File
@@ -691,7 +691,10 @@ class ASameSizeReuploadIsSeen(unittest.TestCase):
def replaced(self, digests: dict, size: int | None) -> bool:
with mock.patch.object(largefiles, "_served_digests", lambda: digests), \
mock.patch.object(largefiles, "_served_size", lambda name: size):
return largefiles._replaced_on_release("dsi_nand.bin", str(self.cached), False)
return largefiles._replaced_on_release(
"dsi_nand.bin", hashlib.sha256(self.cached.read_bytes()).hexdigest(),
self.cached.stat().st_size, False,
)
def test_the_digest_decides(self):
same = hashlib.sha256(b"OLD NAND").hexdigest()
@@ -703,6 +706,30 @@ class ASameSizeReuploadIsSeen(unittest.TestCase):
self.assertFalse(self.replaced({}, 8))
self.assertTrue(self.replaced({}, 9))
def test_a_copy_promoted_during_the_check_is_the_one_judged(self):
"""Another run promotes the served revision while this one hashes the
stale copy: the release check reread the path, found the served
bytes, and the caller was told the cache held no such file."""
new = b"NEW NAND"
real = largefiles.compute_hashes
def promote_while_hashing(path, *args, **kwargs):
result = real(path, *args, **kwargs)
if Path(path).read_bytes() != new:
promoted = self.dir / "promoted.tmp"
promoted.write_bytes(new)
os.replace(promoted, self.cached)
return result
digests = {"dsi_nand.bin": hashlib.sha256(new).hexdigest()}
with mock.patch.object(largefiles, "compute_hashes", promote_while_hashing), \
mock.patch.object(largefiles, "_served_digests", lambda: digests):
found = largefiles._fetch_asset(
"dsi_nand.bin", str(self.dir), hashlib.sha1(new).hexdigest(), "",
offline=False,
)
self.assertEqual(found, str(self.cached))
def test_the_release_check_compares_digests_at_equal_size(self):
expected = {"bios/Nintendo/DSi/dsi_nand.bin": 8}
assets = {"dsi_nand.bin": 8}
+58
View File
@@ -166,5 +166,63 @@ class EmulatorPackHoldsTheCache(unittest.TestCase):
self.assertEqual(held, [True])
class AWaitIsAnnounced(unittest.TestCase):
def test_a_held_tree_is_named_before_the_wait(self):
"""A pipeline holds every tree for its run: the refresh beside it
waited with nothing on screen."""
with tempfile.TemporaryDirectory() as tmp:
cache = Path(tmp) / "probe"
taken = threading.Event()
def second_run() -> None:
with rdd.cache_lock(cache):
taken.set()
with self.assertLogs(rdd.log, "WARNING") as logs:
with rdd.cache_lock(cache):
worker = threading.Thread(target=second_run)
worker.start()
for _ in range(100):
if logs.output:
break
threading.Event().wait(0.05)
self.assertFalse(taken.is_set())
worker.join(10)
self.assertTrue(taken.is_set())
self.assertIn("waiting for", logs.output[0])
self.assertIn("probe", logs.output[0])
class DryRunReport(unittest.TestCase):
"""What a dry run says: why each entry would be fetched, and how many."""
def test_a_quoted_etag_is_cut_without_its_quotes(self):
self.assertEqual(rdd._short_tag('"0x8DF0B72EB52A1C0"'), "0x8DF0B72EB5")
self.assertEqual(rdd._short_tag('W/"2685251384"'), "2685251384")
self.assertEqual(rdd._short_tag(None), "none")
def test_the_remote_tag_is_shown_beside_the_cached_one(self):
with tempfile.TemporaryDirectory() as tmp:
versions = Path(tmp) / "versions.json"
versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}}))
entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip",
"local_cache": tmp}
with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \
self.assertLogs(rdd.log, "INFO") as logs:
self.assertTrue(rdd.refresh_entry("k", entry, dry_run=True,
versions_path=str(versions)))
self.assertIn("cached: 2725165595, remote: 2685251384", logs.output[0])
def test_a_run_ends_on_its_counts(self):
self.assertEqual(
rdd.summarize({"a": True, "b": False, "c": False}, dry_run=True),
"3 entries: 1 would refresh, 2 up to date",
)
self.assertEqual(
rdd.summarize({"a": True, "b": None}, dry_run=False),
"2 entries: 1 refreshed, 0 up to date, 1 failed",
)
if __name__ == "__main__":
unittest.main()
+10
View File
@@ -101,6 +101,16 @@ class Planning(unittest.TestCase):
)
self.assertEqual([f["subject"] for f in surfaced], ["redump"])
def test_a_data_row_outside_the_registry_needs_a_human(self):
"""refresh_data_dirs.py --key 'buildbot system' exits 1 on an unknown
key: the row asks for a registry entry, not a refresh."""
jobs, surfaced, _ = self._plan(
finding("data", "buildbot system"), finding("data", "_data_dirs.yml")
)
self.assertEqual(jobs, [])
self.assertEqual([f["subject"] for f in surfaced],
["buildbot system", "_data_dirs.yml"])
def test_only_stale_rows_run_and_errors_are_shown(self):
jobs, surfaced, ignored = self._plan(
finding("data", "a", "OK"),
+28
View File
@@ -28,7 +28,9 @@ from romset_recipes import ( # noqa: E402
write_reconstructions,
)
from scripts.scraper.romset_dat_importer import ( # noqa: E402
catalog_targets,
compact_entries,
holds_target,
listxml_entries,
git_blob_sha,
merge_snapshot,
@@ -386,6 +388,32 @@ class ListXmlRecipes(unittest.TestCase):
[m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
)
def test_a_set_holding_a_missing_catalog_dump_is_kept(self):
"""fm7740sx carries three No-Intro dumps the collection lacks. No
profile names it, so the missing report named no place to find them."""
targets = {(_crc(ROM_B), len(ROM_B)): {""}}
entries = listxml_entries(
self._opener(self._document()), "MAME 0.289", {"other.zip"}, targets
)
self.assertEqual([entry["name"] for entry in entries], ["childset.zip"])
def test_a_catalog_sha1_must_agree(self):
member = {"crc32": _crc(ROM_B), "size": len(ROM_B), "sha1": "a" * 40}
self.assertFalse(holds_target([member], {(_crc(ROM_B), len(ROM_B)): {"b" * 40}}))
self.assertTrue(holds_target([member], {(_crc(ROM_B), len(ROM_B)): {"a" * 40}}))
def test_catalog_targets_leave_out_what_the_collection_holds(self):
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
root = Path(directory)
(root / "prov").mkdir()
(root / "prov" / "tosec.json").write_text(json.dumps({"source": "tosec", "entries": [
{"name": "held.rom", "crc32": "1", "size": 4, "sha1": "a" * 40},
{"name": "lacked.rom", "crc32": "2", "size": 4, "sha1": "b" * 40},
]}))
(root / "db.json").write_text(json.dumps({"files": {"a" * 40: {}}}))
targets = catalog_targets(str(root / "prov"), str(root / "db.json"))
self.assertEqual(targets, {("00000002", 4): {"b" * 40}})
def test_upstream_blobs_are_recorded_and_replaced(self):
"""The snapshot names the DAT blobs it was read from, so a later
check can tell whether upstream moved without importing again."""
+122
View File
@@ -882,5 +882,127 @@ class OneNativeIdOneSystem(unittest.TestCase):
ids = {_norm_system_id(s) for s in data["systems"]}
self.assertEqual(wanted & ids & machines, wanted)
class HashScraperRefusesAnEmptyRead(unittest.TestCase):
"""A source tree that yields no BIOS set is never cached as a result.
Cached, it was served for a day: every profile entry came out "out of
scope", nothing was refreshed and pipeline steps 2a and 2a2 said OK.
"""
def _fresh(self, sets: dict) -> dict:
from datetime import datetime, timezone # noqa: PLC0415
return {"fetched_at": datetime.now(timezone.utc).isoformat(),
"commit": "abc", "bios_sets": sets}
def test_mame_raises_and_writes_no_cache(self):
from scraper import mame_hash_scraper as m # noqa: PLC0415
with tempfile.TemporaryDirectory() as tmp:
cache = Path(tmp) / "mame-hashes.json"
with mock.patch.object(m, "_CACHE_PATH", cache), \
mock.patch.object(m, "_get_release_tag", return_value="mame0289"), \
mock.patch.object(m, "_sparse_clone"), \
mock.patch.object(m, "_cleanup"), \
mock.patch.object(m, "_get_commit", return_value="abc"), \
mock.patch.object(m, "parse_mame_source_tree", return_value={}):
with self.assertRaises(RuntimeError):
m._fetch_hashes(force=True)
self.assertFalse(cache.exists())
def test_mame_does_not_serve_an_empty_cache(self):
from scraper import mame_hash_scraper as m # noqa: PLC0415
self.assertTrue(m._is_stale(self._fresh({})))
self.assertFalse(m._is_stale(self._fresh({"neogeo": {}})))
def test_fbneo_raises_and_writes_no_cache(self):
from scraper import fbneo_hash_scraper as f # noqa: PLC0415
with tempfile.TemporaryDirectory() as tmp:
cache = Path(tmp) / "fbneo-hashes.json"
with mock.patch.object(f, "CACHE_PATH", cache), \
mock.patch.object(f, "_sparse_clone"), \
mock.patch.object(f, "_cleanup"), \
mock.patch.object(f, "_extract_version", return_value=("v1.0.0.03", "abc")), \
mock.patch.object(f, "parse_fbneo_source_tree", return_value={}):
with self.assertRaises(RuntimeError):
f.fetch_and_cache(force=True)
self.assertFalse(cache.exists())
def test_fbneo_does_not_serve_an_empty_cache(self):
from scraper import fbneo_hash_scraper as f # noqa: PLC0415
with tempfile.TemporaryDirectory() as tmp:
cache = Path(tmp) / "fbneo-hashes.json"
with mock.patch.object(f, "CACHE_PATH", cache):
cache.write_text(json.dumps(self._fresh({})))
self.assertFalse(f._is_cache_fresh())
cache.write_text(json.dumps(self._fresh({"neogeo": {}})))
self.assertTrue(f._is_cache_fresh())
class EmptyScrapeWritesNothing(unittest.TestCase):
"""An upstream that changed shape parses to nothing: the run fails and
the committed platform file stays as it was."""
def _run(self, scraper_class, output: Path) -> int:
from scraper.base_scraper import scraper_cli # noqa: PLC0415
argv = ["scraper", "-o", str(output)]
with mock.patch.object(sys, "argv", argv), \
mock.patch("sys.stderr", new_callable=io.StringIO):
try:
scraper_cli(scraper_class)
except SystemExit as exc:
return int(exc.code or 0)
return 0
def test_no_requirement_exits_non_zero(self):
with tempfile.TemporaryDirectory() as tmp:
output = Path(tmp) / "platform.yml"
output.write_text("systems:\n a:\n files:\n - name: x\n")
self.assertEqual(self._run(_StubScraper, output), 1)
self.assertIn("name: x", output.read_text())
def test_a_generated_file_without_files_exits_non_zero(self):
class NoSystems(_StubScraper):
def __init__(self):
super().__init__([BiosRequirement(name="a.bin", system="s")])
def generate_platform_yaml(self) -> dict:
return {"platform": "x", "systems": {}}
with tempfile.TemporaryDirectory() as tmp:
output = Path(tmp) / "platform.yml"
self.assertEqual(self._run(NoSystems, output), 1)
self.assertFalse(output.exists())
def test_a_platform_that_inherits_its_files_may_list_none(self):
class CoresOnly(_StubScraper):
lists_files = False
def generate_platform_yaml(self) -> dict:
return {"inherits": "retroarch", "cores": ["mgba"]}
with tempfile.TemporaryDirectory() as tmp:
output = Path(tmp) / "platform.yml"
with mock.patch("sys.stdout", new_callable=io.StringIO):
self.assertEqual(self._run(CoresOnly, output), 0)
self.assertIn("mgba", output.read_text())
def test_retropie_without_a_package_raises(self):
archive = RetroPiePackageList._archive(
{"scriptmodules/supplementary/autostart.sh": 'rp_module_id="autostart"\n'}
)
from scraper.retropie_scraper import Scraper # noqa: PLC0415
scraper = Scraper(url="https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "0" * 40)
with self.assertRaises(ValueError):
scraper.module_ids(archive)
if __name__ == "__main__":
unittest.main()
+35 -1
View File
@@ -307,7 +307,7 @@ class RetroPieModuleFlags(unittest.TestCase):
}
def fake_fetch(url, **_kwargs):
section = url.rstrip("/").rsplit("/", 1)[-1]
section = url.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1]
if section in listings:
return json.dumps(listings[section])
return modules[url.split("scriptmodules/", 1)[1]]
@@ -321,6 +321,40 @@ class RetroPieModuleFlags(unittest.TestCase):
class TargetsReadThePlatformRevision(unittest.TestCase):
"""A target list read on master filtered packages by flags the pinned
package list does not have: every request names the platform's pin."""
def _urls(self, module, scraper) -> list[str]:
seen: list[str] = []
def record(url, **_kwargs):
seen.append(url)
if "contents" in url:
return json.dumps([{"name": "lr-mgba.sh"}])
return 'rp_module_id="lr-mgba"'
with mock.patch.object(module, "_fetch", record):
scraper.fetch_targets()
return seen
def test_retropie_reads_its_codeload_pin(self):
pin = retropie_targets_scraper.Scraper.pinned_revision("retropie")
self.assertRegex(pin, r"^[0-9a-f]{40}$")
urls = self._urls(retropie_targets_scraper, retropie_targets_scraper.Scraper())
self.assertTrue(urls)
self.assertTrue(all(pin in url for url in urls), urls)
def test_emudeck_steamos_reads_its_pin(self):
pin = emudeck_targets_scraper.Scraper.pinned_revision("emudeck")
scraper = emudeck_targets_scraper.Scraper()
with mock.patch.object(scraper, "_load_retroarch_cores", return_value=[]):
urls = self._urls(emudeck_targets_scraper, scraper)
steamos = [u for u in urls if "dragoonDorise" in u]
self.assertTrue(steamos)
self.assertTrue(all(u.endswith(f"?ref={pin}") for u in steamos), steamos)
class HashScrapersRefuseAnUnknownVersion(unittest.TestCase):
"""mame_hash_scraper wrote core_version "unknown" after a failed API call
and cached it for a day, while the pipeline step still read OK."""
+11 -3
View File
@@ -561,7 +561,10 @@ today, and answer with one row per subject. Platform and target files are
re-scraped through the scrapers' own write path into a throwaway copy and
diffed against the committed file, so the diff decides rather than a version
string. Core-info is listed and compared with the profiles (a core with no
profile, a profile that calls standalone a core libretro now builds). Data
profile, a profile that calls standalone a core libretro now builds, a
profiled core whose `.info` git blob differs from the one recorded in
`platforms/_core_info.json`). Once the changed `.info` is read against the
profile, `--record-coreinfo <core>` records its blob as reviewed. Data
directories are checked against their upstream, dump catalogs and romset
snapshots against the catalog they were read from (Redump DAT versions, the
No-Intro daily rebuild, the newest TOSEC pack, the newest MAME release, the
@@ -579,6 +582,7 @@ python scripts/check_freshness.py --only platforms,targets
python scripts/check_freshness.py --json
python scripts/check_freshness.py --profiles # also runs profile_sync (slow)
python scripts/check_freshness.py --offline # local ages only
python scripts/check_freshness.py --record-coreinfo virtualjaguar # .info read, record it
```
The exit code is non-zero when anything is stale. Emulator profiles are the
@@ -742,11 +746,15 @@ python -m scripts.scraper.romset_dat_importer --source fbneo --fetch
Snapshots land in `recipes/`, not `provenance/`: the latter holds dump catalogues
and a recipe is not one. Versions accumulate rather than replace each other,
because a platform pins the archive of whichever version its list was built
against. Only sets a profile or platform references are kept, a `romof` parent's
against. Only sets a profile or platform references are kept, plus the sets
holding a dump-catalog entry the collection lacks (crc32 and size, and sha1
when both sides carry one), so `provenance_report.py --missing` can name the
set that ships it: MAME's `fm7740sx` carries the FM77AV40SX `DICROM.rom`,
`EXTSUB.rom` and `KANJI2.rom` that No-Intro lists. A `romof` parent's
members are merged into the child, and members with no CRC32 are dropped: those
are undumped and a real romset does not carry them.
The store is compacted: 22 MAME versions produce 23,679 entries but only 1,726
The store is compacted: 22 MAME versions produce 24,101 entries but only 2,148
distinct recipes, since most sets do not change between releases. Each is kept
once, `dats` listing every version that agrees and `dat` naming the earliest, so
an identification reads as "unchanged since that version".