diff --git a/docs_assets/site.js b/docs_assets/site.js index db4d72e1..33548400 100644 --- a/docs_assets/site.js +++ b/docs_assets/site.js @@ -148,28 +148,88 @@ }); }; - // files/?sha1= is a file's content address: the card it names may - // sit on another manufacturer's page today than when the link was cited. + // files/?hash= reads one shard of the lookup, named by the first two hex + // characters of the value: 20 KB where the whole index was 675 KB. + const HASH_LENGTHS = { 8: "CRC32", 32: "MD5", 40: "SHA1", 64: "SHA256" }; + const LAYER_LABELS = { + collection: "In the collection", + "data-directory": "In a data directory the packs ship", + platform: "Declared by a platform", + emulator: "Read by an emulator", + }; + const siteRoot = () => { + const script = document.querySelector('script[src$="javascripts/site.js"]'); + return script ? new URL("../", script.src) : new URL("../", window.location.href); + }; + const lookupShard = (value) => + fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot())) + .then((response) => (response.ok ? response.json() : { items: [] })) + .then((shard) => shard.items || []); + const normalizeHash = (raw) => raw.trim().toLowerCase().replace(/^0x/, "").replace(/\s+/g, ""); + + const renderMatches = (status, value, items) => { + status.textContent = ""; + const kind = HASH_LENGTHS[value.length]; + const heading = document.createElement("p"); + heading.textContent = `${items.length} ${items.length === 1 ? "match" : "matches"} for ${kind} ${value}:`; + status.append(heading); + Object.entries(LAYER_LABELS).forEach(([layer, label]) => { + const found = items.filter((item) => item.layer === layer); + if (!found.length) return; + const title = document.createElement("p"); + const strong = document.createElement("strong"); + strong.textContent = label; + title.append(strong); + const list = document.createElement("ul"); + found.forEach((item) => { + const row = document.createElement("li"); + const link = document.createElement("a"); + link.href = new URL(item.url, siteRoot()).href; + const code = document.createElement("code"); + code.textContent = item.name || item.value; + link.append(code); + row.append(link); + if (item.owner) row.append(` (${item.owner})`); + list.append(row); + }); + status.append(title, list); + }); + }; + + // files/?sha1= is a file's content address and opens its card + // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and + // lists every page that names the value. const resolveFileAddress = () => { const status = document.getElementById("rb-file-lookup-status"); if (!status) return; - const sha1 = (new URLSearchParams(window.location.search).get("sha1") || "") - .trim().toLowerCase(); - if (!sha1) return; - if (!/^[0-9a-f]{40}$/.test(sha1)) { - status.textContent = "A SHA1 is 40 hexadecimal characters."; + const params = new URLSearchParams(window.location.search); + const permalink = params.has("sha1"); + const value = normalizeHash(params.get("sha1") || params.get("hash") || ""); + if (!value) return; + const input = document.getElementById("rb-hash"); + if (input) input.value = value; + const prefix = permalink && /^[0-9a-f]{12}$/.test(value); + if (!prefix && !(/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length])) { + status.textContent = + "A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters."; return; } status.textContent = "Looking it up..."; - const index = new URL("../api/v1/file-locations.json", window.location.href); - fetch(index) - .then((response) => response.json()) - .then((document_) => { - const item = (document_.items || []).find((entry) => entry.sha1 === sha1); - if (item) { - window.location.replace(new URL("../" + item.url, window.location.href)); + lookupShard(value) + .then((items) => { + const matches = items.filter((item) => + prefix + ? item.algorithm === "sha1" && item.value.startsWith(value) + : item.value === value + ); + const cards = matches.filter((item) => item.layer === "collection" || item.layer === "data-directory"); + const targets = new Set(cards.map((item) => item.url)); + if (permalink && targets.size === 1) { + window.location.replace(new URL(cards[0].url, siteRoot())); + } else if (matches.length) { + renderMatches(status, prefix ? matches[0].value : value, prefix ? cards : matches); } else { - status.textContent = "The collection holds no file with this SHA1."; + status.textContent = "No file the site knows has this hash."; } }) .catch(() => { @@ -177,10 +237,45 @@ }); }; + // A file card moved to another page keeps its address: an id of the form + // sha1-<12 hex> that this page does not hold is looked up by content, and + // an emulator card listed in the table of a split page opens its page. + const followMovedCard = () => { + const id = decodeURIComponent(window.location.hash.slice(1)); + if (!id) return; + const target = document.getElementById(id); + const row = target?.closest(".rb-card-index tr"); + const link = row?.querySelector("a[href]"); + if (link) { + window.location.replace(link.href); + return; + } + const match = /^sha1-([0-9a-f]{12})$/.exec(id); + if (!match || target) return; + window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot())); + }; + + // The site search no longer holds hashes: Enter on a hash opens the lookup. + const hashSearch = () => { + const query = document.querySelector('input[data-md-component="search-query"]'); + if (!query || query.dataset.rbHash === "true") return; + query.dataset.rbHash = "true"; + query.addEventListener("keydown", (event) => { + if (event.key !== "Enter") return; + const value = normalizeHash(query.value); + if (/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length]) { + event.preventDefault(); + window.location.assign(new URL(`files/?hash=${value}`, siteRoot())); + } + }); + }; + const enhance = () => { enhanceTables(); nameMaterialWidgets(); resolveFileAddress(); + followMovedCard(); + hashSearch(); }; let resizeTimer = 0; diff --git a/mkdocs.yml b/mkdocs.yml index 9d1da04b..17260560 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,19 +84,17 @@ markdown_extensions: - tables - toc: permalink: true - # File cards are level 4 headings: the search lands on them, and a - # page of a hundred cards keeps a readable table of contents. + # Level 3 is the wiki's subsections. On pages of file cards, also level + # 3, the hook keeps the table of contents to level 2. toc_depth: 3 - pymdownx.details - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.keys -- pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format +# No mermaid fence: Material drew it in the browser with a script from a +# CDN. Diagrams are SVG files written by scripts/sitediagrams.py. +- pymdownx.superfences - pymdownx.tabbed: alternate_style: true plugins: @@ -245,587 +243,613 @@ nav: - 2600.emu: emulators/2600-emu.md - 3dSen: emulators/3dsen.md - A to C: - - a5200: emulators/a5200.md - - A7800: emulators/a7800.md - - Abuse: emulators/abuse.md - - ACE-DL: emulators/ace-dl.md - - advanced_drastic: emulators/advanced_drastic.md - - AdvanceMAME: emulators/advancemame.md - - AdvanceMAME 0.94: emulators/advmame-0.94.md - - AdvanceMAME 1.4: emulators/advmame-1.4.md - - Adventure Game Studio: emulators/ags.md - - AetherSX2: emulators/aethersx2.md - - Aleph One: emulators/alephone.md - - Altirra: emulators/altirra.md - - amiarcadia: emulators/amiarcadia.md - - Amiberry: emulators/amiberry.md - - AppleWin: emulators/applewin.md - - aPS3e: emulators/aps3e.md - - ArcadeDuck: emulators/arcadeduck.md - - ArcadeFlashWeb: emulators/arcadeflashweb.md - - Ardens: emulators/ardens.md - - Arduous: emulators/arduous.md - - ares: emulators/ares.md - - ARMSX1: emulators/armsx1.md - - ARMSX2: emulators/armsx2.md - - Atari800: emulators/atari800.md - - aX360e: emulators/ax360e.md - - Azahar: emulators/azahar.md - - AzaharPlus: emulators/azaharplus.md - - b2: emulators/b2.md - - Basilisk II: emulators/basiliskii.md - - BBKEmu: emulators/bbkemu.md - - beebem: emulators/beebem.md - - Beetle GBA (Mednafen): emulators/beetle_gba.md - - Beetle Lynx (Mednafen Lynx): emulators/beetle_lynx.md - - Beetle NGP (Mednafen Neo Geo Pocket): emulators/beetle_ngp.md - - Beetle PC-FX (Mednafen): emulators/beetle_pcfx.md - - Beetle PCE (Mednafen PCE): emulators/beetle_pce.md - - Beetle PSX (Mednafen PSX): emulators/beetle_psx.md - - Beetle VB (Mednafen Virtual Boy): emulators/beetle_vb.md - - Beetle WonderSwan (Mednafen WonderSwan): emulators/beetle_wswan.md - - beetle_saturn: emulators/beetle_saturn.md - - BennuGD: emulators/bennugd.md - - BennuGD monolithic (bgdi): emulators/bennugd-monolithic.md - - Bermuda Syndrome: emulators/bsyndrome.md - - Bermuda Syndrome (bs): emulators/bermuda.md - - BigPEmu: emulators/bigpemu.md - - BizHawk: emulators/bizhawk.md - - bk-emulator: emulators/bk.md - - BlastEm: emulators/blastem.md - - blueMSX: emulators/bluemsx.md - - bnes: emulators/bnes.md - - boom3: emulators/boom3.md - - Boytacean: emulators/boytacean.md - - bsnes: emulators/bsnes.md - - bsnes 2014: emulators/bsnes2014.md - - bsnes C++98: emulators/bsnes_cplusplus98.md - - bsnes-hd beta: emulators/bsnes_hd_beta.md - - bsnes-jg: emulators/bsnes-jg.md - - bsnes-mercury: emulators/bsnes_mercury.md - - BStone: emulators/bstone.md - - C-Dogs SDL: emulators/cdogs.md - - C64.emu: emulators/c64-emu.md - - Cannonball: emulators/cannonball.md - - CaPriCe Forever: emulators/capriceforever.md - - Caprice32: emulators/cap32.md - - CapriceRPI: emulators/capricerpi.md - - CatacombGL: emulators/catacombgl.md - - ccleste: emulators/ccleste.md - - CDi 2015: emulators/cdi2015.md - - Cemu: emulators/cemu.md - - Chihiro-DS: emulators/chihiro-ds.md - - ChimeraSNES: emulators/chimerasnes.md - - Chocolate Doom: emulators/chocolate-doom.md - - Citra: emulators/citra.md - - Citra 2018: emulators/citra2018.md - - Citra Canary: emulators/citra_canary.md - - Citra MMJ: emulators/citra-mmj.md - - Citron: emulators/citron.md - - Clock Signal (CLK): emulators/clk.md - - ClownMDEmu: emulators/clownmdemu.md - - ColEm: emulators/colem.md - - Commander Genius: emulators/cgenius.md - - CoolbarEmu: emulators/coolbaremu.md - - CoolCV: emulators/coolcv.md - - corsixth: emulators/corsixth.md - - CPCemu: emulators/cpcemu.md - - CrocoDS: emulators/crocods.md - - Cruzes: emulators/cruzes.md - - CSpect: emulators/cspect.md - - Cxbx-Reloaded: emulators/cxbx-reloaded.md - - Cxbx-Reloaded (31ff15b): emulators/cxbx-reloaded-2023.md + - A: + - a5200: emulators/a5200.md + - A7800: emulators/a7800.md + - Abuse: emulators/abuse.md + - ACE-DL: emulators/ace-dl.md + - advanced_drastic: emulators/advanced_drastic.md + - AdvanceMAME: emulators/advancemame.md + - AdvanceMAME 0.94: emulators/advmame-0.94.md + - AdvanceMAME 1.4: emulators/advmame-1.4.md + - Adventure Game Studio: emulators/ags.md + - AetherSX2: emulators/aethersx2.md + - Aleph One: emulators/alephone.md + - Altirra: emulators/altirra.md + - amiarcadia: emulators/amiarcadia.md + - Amiberry: emulators/amiberry.md + - AppleWin: emulators/applewin.md + - aPS3e: emulators/aps3e.md + - ArcadeDuck: emulators/arcadeduck.md + - ArcadeFlashWeb: emulators/arcadeflashweb.md + - Ardens: emulators/ardens.md + - Arduous: emulators/arduous.md + - ares: emulators/ares.md + - ARMSX1: emulators/armsx1.md + - ARMSX2: emulators/armsx2.md + - Atari800: emulators/atari800.md + - aX360e: emulators/ax360e.md + - Azahar: emulators/azahar.md + - AzaharPlus: emulators/azaharplus.md + - B: + - b2: emulators/b2.md + - Basilisk II: emulators/basiliskii.md + - BBKEmu: emulators/bbkemu.md + - beebem: emulators/beebem.md + - Beetle GBA (Mednafen): emulators/beetle_gba.md + - Beetle Lynx (Mednafen Lynx): emulators/beetle_lynx.md + - Beetle NGP (Mednafen Neo Geo Pocket): emulators/beetle_ngp.md + - Beetle PC-FX (Mednafen): emulators/beetle_pcfx.md + - Beetle PCE (Mednafen PCE): emulators/beetle_pce.md + - Beetle PSX (Mednafen PSX): emulators/beetle_psx.md + - Beetle VB (Mednafen Virtual Boy): emulators/beetle_vb.md + - Beetle WonderSwan (Mednafen WonderSwan): emulators/beetle_wswan.md + - beetle_saturn: emulators/beetle_saturn.md + - BennuGD: emulators/bennugd.md + - BennuGD monolithic (bgdi): emulators/bennugd-monolithic.md + - Bermuda Syndrome: emulators/bsyndrome.md + - Bermuda Syndrome (bs): emulators/bermuda.md + - BigPEmu: emulators/bigpemu.md + - BizHawk: emulators/bizhawk.md + - bk-emulator: emulators/bk.md + - BlastEm: emulators/blastem.md + - blueMSX: emulators/bluemsx.md + - bnes: emulators/bnes.md + - boom3: emulators/boom3.md + - Boytacean: emulators/boytacean.md + - bsnes: emulators/bsnes.md + - bsnes 2014: emulators/bsnes2014.md + - bsnes C++98: emulators/bsnes_cplusplus98.md + - bsnes-hd beta: emulators/bsnes_hd_beta.md + - bsnes-jg: emulators/bsnes-jg.md + - bsnes-mercury: emulators/bsnes_mercury.md + - BStone: emulators/bstone.md + - C: + - C-Dogs SDL: emulators/cdogs.md + - C64.emu: emulators/c64-emu.md + - Cannonball: emulators/cannonball.md + - CaPriCe Forever: emulators/capriceforever.md + - Caprice32: emulators/cap32.md + - CapriceRPI: emulators/capricerpi.md + - CatacombGL: emulators/catacombgl.md + - ccleste: emulators/ccleste.md + - CDi 2015: emulators/cdi2015.md + - Cemu: emulators/cemu.md + - Chihiro-DS: emulators/chihiro-ds.md + - ChimeraSNES: emulators/chimerasnes.md + - Chocolate Doom: emulators/chocolate-doom.md + - Citra: emulators/citra.md + - Citra 2018: emulators/citra2018.md + - Citra Canary: emulators/citra_canary.md + - Citra MMJ: emulators/citra-mmj.md + - Citron: emulators/citron.md + - Clock Signal (CLK): emulators/clk.md + - ClownMDEmu: emulators/clownmdemu.md + - ColEm: emulators/colem.md + - Commander Genius: emulators/cgenius.md + - CoolbarEmu: emulators/coolbaremu.md + - CoolCV: emulators/coolcv.md + - corsixth: emulators/corsixth.md + - CPCemu: emulators/cpcemu.md + - CrocoDS: emulators/crocods.md + - Cruzes: emulators/cruzes.md + - CSpect: emulators/cspect.md + - Cxbx-Reloaded: emulators/cxbx-reloaded.md + - Cxbx-Reloaded (31ff15b): emulators/cxbx-reloaded-2023.md - D to F: - - D.Smile: emulators/dsmile.md - - DaedalusX64: emulators/daedalusx64.md - - Daphne: emulators/daphne.md - - DarkPlaces: emulators/darkplaces.md - - Demul: emulators/demul.md - - DeSmuME: emulators/desmume.md - - DeSmuME 2015: emulators/desmume2015.md - - devilutionx: emulators/devilutionx.md - - DGen/SDL: emulators/dgen.md - - dhewm3: emulators/dhewm3.md - - DICE: emulators/dice.md - - DingooEmu: emulators/dingooemu.md - - Dinothawr: emulators/dinothawr.md - - DirectXBox: emulators/directxbox.md - - DirkSimple: emulators/dirksimple.md - - Dolphin: emulators/dolphin.md - - Dolphin Launcher: emulators/dolphin_launcher.md - - Dolphin MMJR: emulators/dolphin-mmjr.md - - Dolphin MMJR2: emulators/dolphin-mmjr2.md - - DOSBox: emulators/dosbox.md - - DOSBox (SDL2 fork): emulators/dosbox-sdl2.md - - DOSBox Pure: emulators/dosbox_pure.md - - DOSBox Pure Unleashed: emulators/dosbox-pure-unleashed.md - - DOSBox Staging: emulators/dosbox-staging.md - - DOSBox-core: emulators/dosbox_core.md - - DOSBox-SVN: emulators/dosbox_svn.md - - DOSBox-SVN CE: emulators/dosbox_svn_ce.md - - DOSBox-X: emulators/dosbox-x.md - - DoubleCherryGB: emulators/doublecherrygb.md - - doukutsu-rs: emulators/doukutsu_rs.md - - DraStic: emulators/drastic.md - - DREAMM: emulators/dreamm.md - - DroidArcadia: emulators/droidarcadia.md - - dsda-doom: emulators/dsda-doom.md - - DSperate: emulators/dsperate.md - - DuckStation: emulators/duckstation.md - - DuckStation (legacy): emulators/duckstation-legacy.md - - Dusklight: emulators/dusklight.md - - dxx-rebirth: emulators/dxx-rebirth.md - - EasyRPG Player: emulators/easyrpg.md - - ECWolf: emulators/ecwolf.md - - Eden: emulators/eden.md - - EDuke32: emulators/eduke32.md - - EightyOne: emulators/81.md - - EightyOne (Recalbox): emulators/eightyone.md - - eka2l1: emulators/eka2l1.md - - EmuCoreV: emulators/emucorev.md - - EmuCoreX: emulators/emucorex.md - - EmuCV: emulators/emucv.md - - EmuSCV: emulators/emuscv.md - - emux (CHIP-8): emulators/emux_chip8.md - - emux (Game Boy): emulators/emux_gb.md - - emux (NES): emulators/emux_nes.md - - emux (SMS): emulators/emux_sms.md - - ep128emu-core: emulators/ep128emu.md - - ep128emu_core: emulators/ep128emu_core.md - - ePSXe: emulators/epsxe.md - - ET Legacy: emulators/etlegacy.md - - Eternal Computer: emulators/eternal.md - - Exel-Hence: emulators/exel-hence.md - - FAKE-08: emulators/fake08.md - - fallout1-ce: emulators/fallout1-ce.md - - fallout2-ce: emulators/fallout2-ce.md - - FB Alpha 2012: emulators/fbalpha2012.md - - FB Alpha 2012 CPS-1: emulators/fbalpha2012_cps1.md - - FB Alpha 2012 CPS-2: emulators/fbalpha2012_cps2.md - - FB Alpha 2012 CPS-3: emulators/fbalpha2012_cps3.md - - FB Alpha 2012 Neo Geo: emulators/fbalpha2012_neogeo.md - - FB Alpha 2019: emulators/fbalpha2019.md - - FBZX: emulators/fbzx.md - - FCEUmm: emulators/fceumm.md - - FCEUmm Mod: emulators/fceumm_mod.md - - fheroes2: emulators/fheroes2.md - - FinalBurn Neo: emulators/fbneo.md - - FinalBurn Neo (CPS-1/CPS-2): emulators/fbneo_cps12.md - - FinalBurn Neo (Neo Geo): emulators/fbneo_neogeo.md - - fixGB: emulators/fixgb.md - - fixNES: emulators/fixnes.md - - Flycast: emulators/flycast.md - - Flycast 2021: emulators/flycast2021.md - - Flycast 2022 Low-End: emulators/flycast2021le.md - - Flycast Dojo: emulators/flycast-dojo.md - - fMSX: emulators/fmsx.md - - FPse: emulators/fpse.md - - FPseNG: emulators/fpse-ng.md - - FreeChaF: emulators/freechaf.md - - FreeIntv: emulators/freeintv.md - - FreeJ2ME: emulators/freej2me.md - - FreeJ2ME-Plus: emulators/freej2me_plus.md - - Frodo: emulators/frodo.md - - Frotz: emulators/frotz.md - - FS-UAE: emulators/fsuae.md - - FS-UAE 3.1: emulators/fs-uae.md - - Fuse: emulators/fuse.md - - future-pinball: emulators/future-pinball.md - - future-pinball-fploader: emulators/future-pinball-fploader.md + - D: + - D.Smile: emulators/dsmile.md + - DaedalusX64: emulators/daedalusx64.md + - Daphne: emulators/daphne.md + - DarkPlaces: emulators/darkplaces.md + - Demul: emulators/demul.md + - DeSmuME: emulators/desmume.md + - DeSmuME 2015: emulators/desmume2015.md + - devilutionx: emulators/devilutionx.md + - DGen/SDL: emulators/dgen.md + - dhewm3: emulators/dhewm3.md + - DICE: emulators/dice.md + - DingooEmu: emulators/dingooemu.md + - Dinothawr: emulators/dinothawr.md + - DirectXBox: emulators/directxbox.md + - DirkSimple: emulators/dirksimple.md + - Dolphin: emulators/dolphin.md + - Dolphin Launcher: emulators/dolphin_launcher.md + - Dolphin MMJR: emulators/dolphin-mmjr.md + - Dolphin MMJR2: emulators/dolphin-mmjr2.md + - DOSBox: emulators/dosbox.md + - DOSBox (SDL2 fork): emulators/dosbox-sdl2.md + - DOSBox Pure: emulators/dosbox_pure.md + - DOSBox Pure Unleashed: emulators/dosbox-pure-unleashed.md + - DOSBox Staging: emulators/dosbox-staging.md + - DOSBox-core: emulators/dosbox_core.md + - DOSBox-SVN: emulators/dosbox_svn.md + - DOSBox-SVN CE: emulators/dosbox_svn_ce.md + - DOSBox-X: emulators/dosbox-x.md + - DoubleCherryGB: emulators/doublecherrygb.md + - doukutsu-rs: emulators/doukutsu_rs.md + - DraStic: emulators/drastic.md + - DREAMM: emulators/dreamm.md + - DroidArcadia: emulators/droidarcadia.md + - dsda-doom: emulators/dsda-doom.md + - DSperate: emulators/dsperate.md + - DuckStation: emulators/duckstation.md + - DuckStation (legacy): emulators/duckstation-legacy.md + - Dusklight: emulators/dusklight.md + - dxx-rebirth: emulators/dxx-rebirth.md + - E: + - EasyRPG Player: emulators/easyrpg.md + - ECWolf: emulators/ecwolf.md + - Eden: emulators/eden.md + - EDuke32: emulators/eduke32.md + - EightyOne: emulators/81.md + - EightyOne (Recalbox): emulators/eightyone.md + - eka2l1: emulators/eka2l1.md + - EmuCoreV: emulators/emucorev.md + - EmuCoreX: emulators/emucorex.md + - EmuCV: emulators/emucv.md + - EmuSCV: emulators/emuscv.md + - emux (CHIP-8): emulators/emux_chip8.md + - emux (Game Boy): emulators/emux_gb.md + - emux (NES): emulators/emux_nes.md + - emux (SMS): emulators/emux_sms.md + - ep128emu-core: emulators/ep128emu.md + - ep128emu_core: emulators/ep128emu_core.md + - ePSXe: emulators/epsxe.md + - ET Legacy: emulators/etlegacy.md + - Eternal Computer: emulators/eternal.md + - Exel-Hence: emulators/exel-hence.md + - F: + - FAKE-08: emulators/fake08.md + - fallout1-ce: emulators/fallout1-ce.md + - fallout2-ce: emulators/fallout2-ce.md + - FB Alpha 2012: emulators/fbalpha2012.md + - FB Alpha 2012 CPS-1: emulators/fbalpha2012_cps1.md + - FB Alpha 2012 CPS-2: emulators/fbalpha2012_cps2.md + - FB Alpha 2012 CPS-3: emulators/fbalpha2012_cps3.md + - FB Alpha 2012 Neo Geo: emulators/fbalpha2012_neogeo.md + - FB Alpha 2019: emulators/fbalpha2019.md + - FBZX: emulators/fbzx.md + - FCEUmm: emulators/fceumm.md + - FCEUmm Mod: emulators/fceumm_mod.md + - fheroes2: emulators/fheroes2.md + - FinalBurn Neo: emulators/fbneo.md + - FinalBurn Neo (CPS-1/CPS-2): emulators/fbneo_cps12.md + - FinalBurn Neo (Neo Geo): emulators/fbneo_neogeo.md + - fixGB: emulators/fixgb.md + - fixNES: emulators/fixnes.md + - Flycast: emulators/flycast.md + - Flycast 2021: emulators/flycast2021.md + - Flycast 2022 Low-End: emulators/flycast2021le.md + - Flycast Dojo: emulators/flycast-dojo.md + - fMSX: emulators/fmsx.md + - FPse: emulators/fpse.md + - FPseNG: emulators/fpse-ng.md + - FreeChaF: emulators/freechaf.md + - FreeIntv: emulators/freeintv.md + - FreeJ2ME: emulators/freej2me.md + - FreeJ2ME-Plus: emulators/freej2me_plus.md + - Frodo: emulators/frodo.md + - Frotz: emulators/frotz.md + - FS-UAE: emulators/fsuae.md + - FS-UAE 3.1: emulators/fs-uae.md + - Fuse: emulators/fuse.md + - future-pinball: emulators/future-pinball.md + - future-pinball-fploader: emulators/future-pinball-fploader.md - G to L: - - galaksija: emulators/galaksija.md - - GAM4980: emulators/gam4980.md - - Gambatte: emulators/gambatte.md - - Game & Watch: emulators/gw.md - - GameTank: emulators/gametank.md - - Gargoyle: emulators/gargoyle.md - - GBA.emu: emulators/gba-emu.md - - GBC.emu: emulators/gbc-emu.md - - Gearboy: emulators/gearboy.md - - Gearcoleco: emulators/gearcoleco.md - - Geargrafx: emulators/geargrafx.md - - Gearlynx: emulators/gearlynx.md - - Gearsystem: emulators/gearsystem.md - - Geartowns: emulators/geartowns.md - - GemRB: emulators/gemrb.md - - Genesis Plus GX: emulators/genesis_plus_gx.md - - Genesis Plus GX Expanded: emulators/genesis_plus_gx_expanded.md - - Genesis Plus GX Wide: emulators/genesis_plus_gx_wide.md - - Geolith: emulators/geolith.md - - Ghostship: emulators/ghostship.md - - GnGeo-Pi: emulators/gngeopi.md - - Gopher2600: emulators/gopher2600.md - - gopher64: emulators/gopher64.md - - gpSP: emulators/gpsp.md - - GroovyMAME: emulators/groovymame.md - - GSplus: emulators/gsplus.md - - GSplus (legacy): emulators/gsplus-legacy.md - - GZDoom: emulators/gzdoom.md - - hakuX: emulators/hakux.md - - Handy: emulators/handy.md - - Hatari: emulators/hatari.md - - Hatari 2014: emulators/hatari2014.md - - hatariB: emulators/hatarib.md - - HBMAME (Homebrew MAME): emulators/hbmame.md - - higan (SFC Accuracy): emulators/higan_sfc.md - - hode: emulators/hode.md - - Holani: emulators/holani.md - - Hurrican: emulators/hurrican.md - - Hydra Castle Labyrinth: emulators/hcl.md - - Hypseus Singe: emulators/hypseus-singe.md - - idTech4A++: emulators/idtech4a++.md - - Ikemen GO: emulators/ikemen.md - - iNES: emulators/ines.md - - Infinity: emulators/infinity.md - - Ion Fury: emulators/fury.md - - ioquake3: emulators/ioquake3.md - - ioRTCW: emulators/iortcw.md - - IrataJaguar: emulators/iratajaguar.md - - IroGB: emulators/irogb.md - - Ishiiruka: emulators/ishiiruka.md - - izapple2: emulators/izapple2.md - - J2ME Loader: emulators/j2me-loader.md - - JAXE: emulators/jaxe.md - - Jazz² Resurrection: emulators/jazz2-native.md - - jgenesis: emulators/jgenesis.md - - JL-Mod: emulators/jl-mod.md - - JollyCV: emulators/jollycv.md - - Julius: emulators/julius.md - - Jynx: emulators/jynx.md - - jzIntv: emulators/jzintv.md - - Kega Fusion: emulators/kega-fusion.md - - KEGS: emulators/kegs.md - - KEmulator nnmod: emulators/kemulator.md - - Kenji-NX: emulators/kenji-nx.md - - Kronos: emulators/kronos.md - - Lexaloffle: emulators/lexaloffle.md - - lightspark: emulators/lightspark.md - - Lime3DS: emulators/lime3ds.md - - LinApple: emulators/linapple.md - - Lindbergh Loader: emulators/lindbergh-loader.md - - Linkboy: emulators/linkboy.md - - Linux Loader: emulators/linuxloader.md - - LowRes NX: emulators/lowresnx.md - - LRPS2: emulators/lrps2.md - - Lynx.emu: emulators/lynx-emu.md - - LZDoom: emulators/lzdoom.md + - G: + - galaksija: emulators/galaksija.md + - GAM4980: emulators/gam4980.md + - Gambatte: emulators/gambatte.md + - Game & Watch: emulators/gw.md + - GameTank: emulators/gametank.md + - Gargoyle: emulators/gargoyle.md + - GBA.emu: emulators/gba-emu.md + - GBC.emu: emulators/gbc-emu.md + - Gearboy: emulators/gearboy.md + - Gearcoleco: emulators/gearcoleco.md + - Geargrafx: emulators/geargrafx.md + - Gearlynx: emulators/gearlynx.md + - Gearsystem: emulators/gearsystem.md + - Geartowns: emulators/geartowns.md + - GemRB: emulators/gemrb.md + - Genesis Plus GX: emulators/genesis_plus_gx.md + - Genesis Plus GX Expanded: emulators/genesis_plus_gx_expanded.md + - Genesis Plus GX Wide: emulators/genesis_plus_gx_wide.md + - Geolith: emulators/geolith.md + - Ghostship: emulators/ghostship.md + - GnGeo-Pi: emulators/gngeopi.md + - Gopher2600: emulators/gopher2600.md + - gopher64: emulators/gopher64.md + - gpSP: emulators/gpsp.md + - GroovyMAME: emulators/groovymame.md + - GSplus: emulators/gsplus.md + - GSplus (legacy): emulators/gsplus-legacy.md + - GZDoom: emulators/gzdoom.md + - H: + - hakuX: emulators/hakux.md + - Handy: emulators/handy.md + - Hatari: emulators/hatari.md + - Hatari 2014: emulators/hatari2014.md + - hatariB: emulators/hatarib.md + - HBMAME (Homebrew MAME): emulators/hbmame.md + - higan (SFC Accuracy): emulators/higan_sfc.md + - hode: emulators/hode.md + - Holani: emulators/holani.md + - Hurrican: emulators/hurrican.md + - Hydra Castle Labyrinth: emulators/hcl.md + - Hypseus Singe: emulators/hypseus-singe.md + - I: + - idTech4A++: emulators/idtech4a++.md + - Ikemen GO: emulators/ikemen.md + - iNES: emulators/ines.md + - Infinity: emulators/infinity.md + - Ion Fury: emulators/fury.md + - ioquake3: emulators/ioquake3.md + - ioRTCW: emulators/iortcw.md + - IrataJaguar: emulators/iratajaguar.md + - IroGB: emulators/irogb.md + - Ishiiruka: emulators/ishiiruka.md + - izapple2: emulators/izapple2.md + - J: + - J2ME Loader: emulators/j2me-loader.md + - JAXE: emulators/jaxe.md + - Jazz² Resurrection: emulators/jazz2-native.md + - jgenesis: emulators/jgenesis.md + - JL-Mod: emulators/jl-mod.md + - JollyCV: emulators/jollycv.md + - Julius: emulators/julius.md + - Jynx: emulators/jynx.md + - jzIntv: emulators/jzintv.md + - K: + - Kega Fusion: emulators/kega-fusion.md + - KEGS: emulators/kegs.md + - KEmulator nnmod: emulators/kemulator.md + - Kenji-NX: emulators/kenji-nx.md + - Kronos: emulators/kronos.md + - L: + - Lexaloffle: emulators/lexaloffle.md + - lightspark: emulators/lightspark.md + - Lime3DS: emulators/lime3ds.md + - LinApple: emulators/linapple.md + - Lindbergh Loader: emulators/lindbergh-loader.md + - Linkboy: emulators/linkboy.md + - Linux Loader: emulators/linuxloader.md + - LowRes NX: emulators/lowresnx.md + - LRPS2: emulators/lrps2.md + - Lynx.emu: emulators/lynx-emu.md + - LZDoom: emulators/lzdoom.md - M to O: - - M2000: emulators/m2000.md - - M64Plus FZ: emulators/m64plus-fz.md - - MagicEngine: emulators/magicengine.md - - MAME: emulators/mame.md - - MAME 0.258: emulators/mame0258.md - - MAME 0.278: emulators/mame0278.md - - MAME 2000: emulators/mame2000.md - - MAME 2003: emulators/mame2003.md - - MAME 2003 Midway: emulators/mame2003_midway.md - - MAME 2003-Plus: emulators/mame2003_plus.md - - MAME 2009: emulators/mame2009.md - - MAME 2010: emulators/mame2010.md - - MAME 2014: emulators/mame2014.md - - MAME 2015: emulators/mame2015.md - - MAME 2016: emulators/mame2016.md - - MAME Arcade: emulators/mamearcade.md - - MAME MESS: emulators/mamemess.md - - MAME4ALL-Pi: emulators/mame4all.md - - MAME4droid (0.139u1): emulators/mame4droid.md - - MAME4droid 2024: emulators/mame4droid-2024.md - - Mandarine: emulators/mandarine.md - - Mariani: emulators/mariani.md - - MasterGear: emulators/mastergear.md - - MD.emu: emulators/md-emu.md - - Mednafen: emulators/mednafen.md - - Mednafen STV: emulators/mednafen_stv.md - - mednafen_gba: emulators/mednafen_gba.md - - mednafen_pce: emulators/mednafen_pce.md - - mednafen_pce_fast: emulators/mednafen_pce_fast.md - - mednafen_snes: emulators/mednafen_snes.md - - mednafen_supafaust: emulators/mednafen_supafaust.md - - mednafen_supergrafx: emulators/mednafen_supergrafx.md - - melonDS: emulators/melonds.md - - melonDS DS: emulators/melonds_ds.md - - melonDS Nightly: emulators/melonds-nightly.md - - Mesen: emulators/mesen.md - - mesen-s: emulators/mesen-s.md - - MesenCE: emulators/mesence.md - - MESS 2015: emulators/mess2015.md - - Meteor GBA: emulators/meteor.md - - MFME: emulators/mfme.md - - mGBA: emulators/mgba.md - - mGBA Splitscreen: emulators/mgba_splitscreen.md - - Mini vMac: emulators/minivmac.md - - Mini vMac 37: emulators/minivmac37.md - - mkxp-z: emulators/mkxp_z.md - - Model 2 Emulator: emulators/model2.md - - MojoZork: emulators/mojozork.md - - MSX.emu: emulators/msx-emu.md - - Mu: emulators/mu.md - - Mupen64Plus: emulators/mupen64plus.md - - Mupen64Plus (libretro, 2019): emulators/mupen64plus-libretro.md - - Mupen64Plus AE: emulators/mupen64plus-ae.md - - Mupen64Plus-Next: emulators/mupen64plus_next.md - - My Boy!: emulators/my-boy.md - - My OldBoy!: emulators/my-oldboy.md - - NanoBoyAdvance: emulators/nanoboyadvance.md - - Native32Emu: emulators/native32emu.md - - nekop2: emulators/nekop2.md - - NEO.emu: emulators/neo-emu.md - - NeoCD: emulators/neocd.md - - nes: emulators/nes.md - - NES.emu: emulators/nes-emu.md - - Nesoid: emulators/nesoid.md - - Nestopia UE: emulators/nestopia.md - - NetherSX2: emulators/nethersx2.md - - NetherSX2-Turnip: emulators/nethersx2-turnip.md - - NetherSX2-Turnip Classic: emulators/nethersx2-turnip-classic.md - - NGP.emu: emulators/ngp-emu.md - - NicaiEmu: emulators/nicaiemu.md - - no$gba: emulators/nosgba.md - - NooDS: emulators/noods.md - - NP2kai: emulators/np2kai.md - - np2pi: emulators/np2pi.md - - nSide (SFC Balanced): emulators/higan_sfc_balanced.md - - Nuance: emulators/nuance.md - - Numero: emulators/numero.md - - NXEngine: emulators/nxengine.md - - O2EM: emulators/o2em.md - - Oberon: emulators/oberon.md - - Omicron: emulators/omicron.md - - ONScripter: emulators/onscripter.md - - ONScripter Yuri: emulators/onsyuri.md - - openbor: emulators/openbor.md - - OpenGOAL: emulators/opengoal.md - - OpenJazz: emulators/openjazz.md - - openjk: emulators/openjk.md - - OpenJKDF2: emulators/openjkdf2.md - - OpenLara: emulators/openlara.md - - OpenMoHAA: emulators/openmohaa.md - - openMSX: emulators/openmsx.md - - OpenTTD: emulators/openttd.md - - OpenTyrian: emulators/opentyrian.md - - Opera (4DO): emulators/opera.md - - Oricutron: emulators/oricutron.md - - Osmose (RetroPie): emulators/osmose.md + - M: + - M2000: emulators/m2000.md + - M64Plus FZ: emulators/m64plus-fz.md + - MagicEngine: emulators/magicengine.md + - MAME: emulators/mame.md + - MAME 0.258: emulators/mame0258.md + - MAME 0.278: emulators/mame0278.md + - MAME 2000: emulators/mame2000.md + - MAME 2003: emulators/mame2003.md + - MAME 2003 Midway: emulators/mame2003_midway.md + - MAME 2003-Plus: emulators/mame2003_plus.md + - MAME 2009: emulators/mame2009.md + - MAME 2010: emulators/mame2010.md + - MAME 2014: emulators/mame2014.md + - MAME 2015: emulators/mame2015.md + - MAME 2016: emulators/mame2016.md + - MAME Arcade: emulators/mamearcade.md + - MAME MESS: emulators/mamemess.md + - MAME4ALL-Pi: emulators/mame4all.md + - MAME4droid (0.139u1): emulators/mame4droid.md + - MAME4droid 2024: emulators/mame4droid-2024.md + - Mandarine: emulators/mandarine.md + - Mariani: emulators/mariani.md + - MasterGear: emulators/mastergear.md + - MD.emu: emulators/md-emu.md + - Mednafen: emulators/mednafen.md + - Mednafen STV: emulators/mednafen_stv.md + - mednafen_gba: emulators/mednafen_gba.md + - mednafen_pce: emulators/mednafen_pce.md + - mednafen_pce_fast: emulators/mednafen_pce_fast.md + - mednafen_snes: emulators/mednafen_snes.md + - mednafen_supafaust: emulators/mednafen_supafaust.md + - mednafen_supergrafx: emulators/mednafen_supergrafx.md + - melonDS: emulators/melonds.md + - melonDS DS: emulators/melonds_ds.md + - melonDS Nightly: emulators/melonds-nightly.md + - Mesen: emulators/mesen.md + - mesen-s: emulators/mesen-s.md + - MesenCE: emulators/mesence.md + - MESS 2015: emulators/mess2015.md + - Meteor GBA: emulators/meteor.md + - MFME: emulators/mfme.md + - mGBA: emulators/mgba.md + - mGBA Splitscreen: emulators/mgba_splitscreen.md + - Mini vMac: emulators/minivmac.md + - Mini vMac 37: emulators/minivmac37.md + - mkxp-z: emulators/mkxp_z.md + - Model 2 Emulator: emulators/model2.md + - MojoZork: emulators/mojozork.md + - MSX.emu: emulators/msx-emu.md + - Mu: emulators/mu.md + - Mupen64Plus: emulators/mupen64plus.md + - Mupen64Plus (libretro, 2019): emulators/mupen64plus-libretro.md + - Mupen64Plus AE: emulators/mupen64plus-ae.md + - Mupen64Plus-Next: emulators/mupen64plus_next.md + - My Boy!: emulators/my-boy.md + - My OldBoy!: emulators/my-oldboy.md + - N: + - NanoBoyAdvance: emulators/nanoboyadvance.md + - Native32Emu: emulators/native32emu.md + - nekop2: emulators/nekop2.md + - NEO.emu: emulators/neo-emu.md + - NeoCD: emulators/neocd.md + - nes: emulators/nes.md + - NES.emu: emulators/nes-emu.md + - Nesoid: emulators/nesoid.md + - Nestopia UE: emulators/nestopia.md + - NetherSX2: emulators/nethersx2.md + - NetherSX2-Turnip: emulators/nethersx2-turnip.md + - NetherSX2-Turnip Classic: emulators/nethersx2-turnip-classic.md + - NGP.emu: emulators/ngp-emu.md + - NicaiEmu: emulators/nicaiemu.md + - no$gba: emulators/nosgba.md + - NooDS: emulators/noods.md + - NP2kai: emulators/np2kai.md + - np2pi: emulators/np2pi.md + - nSide (SFC Balanced): emulators/higan_sfc_balanced.md + - Nuance: emulators/nuance.md + - Numero: emulators/numero.md + - NXEngine: emulators/nxengine.md + - O: + - O2EM: emulators/o2em.md + - Oberon: emulators/oberon.md + - Omicron: emulators/omicron.md + - ONScripter: emulators/onscripter.md + - ONScripter Yuri: emulators/onsyuri.md + - openbor: emulators/openbor.md + - OpenGOAL: emulators/opengoal.md + - OpenJazz: emulators/openjazz.md + - openjk: emulators/openjk.md + - OpenJKDF2: emulators/openjkdf2.md + - OpenLara: emulators/openlara.md + - OpenMoHAA: emulators/openmohaa.md + - openMSX: emulators/openmsx.md + - OpenTTD: emulators/openttd.md + - OpenTyrian: emulators/opentyrian.md + - Opera (4DO): emulators/opera.md + - Oricutron: emulators/oricutron.md + - Osmose (RetroPie): emulators/osmose.md - P to R: - - P/EMU/SDL: emulators/piemu.md - - Panda3DS: emulators/panda3ds.md - - Parallel Launcher: emulators/parallel-launcher.md - - ParaLLEl N64: emulators/parallel_n64.md - - pce-emu: emulators/pce-emu.md - - PCEE2: emulators/pcee2.md - - PCem: emulators/pcem.md - - PCSX-ReARMed: emulators/pcsx_rearmed.md - - PCSX-ReARMed (standalone): emulators/pcsx_rearmed_standalone.md - - PCSX1: emulators/pcsx1.md - - PCSX2: emulators/pcsx2.md - - PCSX2 Legacy: emulators/pcsx2-legacy.md - - PCSX2x6: emulators/pcsx2x6.md - - PD777: emulators/pd777.md - - Perfect Dark (PC port): emulators/pdark.md - - Phoenix: emulators/phoenix.md - - Picocomputer 6502: emulators/rp6502.md - - PicoDrive: emulators/picodrive.md - - piece-emu: emulators/piece.md - - piFBA (fba2x): emulators/fba2x.md - - PiSNES: emulators/pisnes.md - - Pizza Boy GBA: emulators/pizza-boy-gba.md - - Pizza Boy GBC: emulators/pizza-boy-gbc.md - - Pizza Boy SC: emulators/pizza-boy-sc.md - - Plastic: emulators/plastic.md - - play: emulators/play.md - - PlaydiaEmu: emulators/playdiaemu.md - - PokeMini: emulators/pokemini.md - - pokketstation: emulators/pokketstation.md - - POM2: emulators/pom2.md - - Potator: emulators/potator.md - - PPSSPP: emulators/ppsspp.md - - PPSSPP 1.5.4: emulators/ppsspp-1.5.4.md - - PrBoom: emulators/prboom.md - - PrBoom+: emulators/prboom-plus.md - - PrimeHack: emulators/primehack.md - - project64: emulators/project64.md - - ProSystem: emulators/prosystem.md - - pSxMAME: emulators/psxmame.md - - PUAE (P-UAE): emulators/puae.md - - PUAE 2021: emulators/puae2021.md - - puNES: emulators/punes.md - - px68k: emulators/px68k.md - - px68k (SDL): emulators/px68k-sdl.md - - QEMU: emulators/qemu.md - - Quake 3 (Raspberry Pi): emulators/quake3.md - - QUASI88: emulators/quasi88.md - - QUASI88kai: emulators/quasi88kai.md - - QuickNES: emulators/quicknes.md - - RACE (Neo Geo Pocket): emulators/race.md - - radio: emulators/radio.md - - RAINE: emulators/raine.md - - Raze: emulators/raze.md - - RB5000: emulators/rb5000.md - - Real3DOPlayer: emulators/real3doplayer.md - - RealRTCW: emulators/realrtcw.md - - Redream: emulators/redream.md - - Reicast: emulators/reicast.md - - REminiscence: emulators/reminiscence.md - - Retro8: emulators/retro8.md - - RetroDream: emulators/retrodream.md - - RigelEngine: emulators/rigelengine.md - - Rosalie's Mupen GUI: emulators/rosalies-mupen-gui.md - - RPCS3: emulators/rpcs3.md - - rpix86: emulators/rpix86.md - - ruffle: emulators/ruffle.md - - Rust-DOS: emulators/rust_dos.md - - Rustation: emulators/rustation.md - - RustyNES: emulators/rustynes.md - - RVVM: emulators/rvvm.md - - Ryujinx: emulators/ryujinx.md + - P: + - P/EMU/SDL: emulators/piemu.md + - Panda3DS: emulators/panda3ds.md + - Parallel Launcher: emulators/parallel-launcher.md + - ParaLLEl N64: emulators/parallel_n64.md + - pce-emu: emulators/pce-emu.md + - PCEE2: emulators/pcee2.md + - PCem: emulators/pcem.md + - PCSX-ReARMed: emulators/pcsx_rearmed.md + - PCSX-ReARMed (standalone): emulators/pcsx_rearmed_standalone.md + - PCSX1: emulators/pcsx1.md + - PCSX2: emulators/pcsx2.md + - PCSX2 Legacy: emulators/pcsx2-legacy.md + - PCSX2x6: emulators/pcsx2x6.md + - PD777: emulators/pd777.md + - Perfect Dark (PC port): emulators/pdark.md + - Phoenix: emulators/phoenix.md + - Picocomputer 6502: emulators/rp6502.md + - PicoDrive: emulators/picodrive.md + - piece-emu: emulators/piece.md + - piFBA (fba2x): emulators/fba2x.md + - PiSNES: emulators/pisnes.md + - Pizza Boy GBA: emulators/pizza-boy-gba.md + - Pizza Boy GBC: emulators/pizza-boy-gbc.md + - Pizza Boy SC: emulators/pizza-boy-sc.md + - Plastic: emulators/plastic.md + - play: emulators/play.md + - PlaydiaEmu: emulators/playdiaemu.md + - PokeMini: emulators/pokemini.md + - pokketstation: emulators/pokketstation.md + - POM2: emulators/pom2.md + - Potator: emulators/potator.md + - PPSSPP: emulators/ppsspp.md + - PPSSPP 1.5.4: emulators/ppsspp-1.5.4.md + - PrBoom: emulators/prboom.md + - PrBoom+: emulators/prboom-plus.md + - PrimeHack: emulators/primehack.md + - project64: emulators/project64.md + - ProSystem: emulators/prosystem.md + - pSxMAME: emulators/psxmame.md + - PUAE (P-UAE): emulators/puae.md + - PUAE 2021: emulators/puae2021.md + - puNES: emulators/punes.md + - px68k: emulators/px68k.md + - px68k (SDL): emulators/px68k-sdl.md + - Q: + - QEMU: emulators/qemu.md + - Quake 3 (Raspberry Pi): emulators/quake3.md + - QUASI88: emulators/quasi88.md + - QUASI88kai: emulators/quasi88kai.md + - QuickNES: emulators/quicknes.md + - R: + - RACE (Neo Geo Pocket): emulators/race.md + - radio: emulators/radio.md + - RAINE: emulators/raine.md + - Raze: emulators/raze.md + - RB5000: emulators/rb5000.md + - Real3DOPlayer: emulators/real3doplayer.md + - RealRTCW: emulators/realrtcw.md + - Redream: emulators/redream.md + - Reicast: emulators/reicast.md + - REminiscence: emulators/reminiscence.md + - Retro8: emulators/retro8.md + - RetroDream: emulators/retrodream.md + - RigelEngine: emulators/rigelengine.md + - Rosalie's Mupen GUI: emulators/rosalies-mupen-gui.md + - RPCS3: emulators/rpcs3.md + - rpix86: emulators/rpix86.md + - ruffle: emulators/ruffle.md + - Rust-DOS: emulators/rust_dos.md + - Rustation: emulators/rustation.md + - RustyNES: emulators/rustynes.md + - RVVM: emulators/rvvm.md + - Ryujinx: emulators/ryujinx.md - S to U: - - SAME CDi: emulators/same_cdi.md - - SameBoy: emulators/sameboy.md - - sameduck: emulators/sameduck.md - - Saturn.emu: emulators/saturn-emu.md - - ScummVM: emulators/scummvm.md - - ScummVM (SDL1): emulators/scummvm-sdl1.md - - SDL2TRS: emulators/sdl2trs.md - - SDLPAL: emulators/sdlpal.md - - sdlpop: emulators/sdlpop.md - - shadps4: emulators/shadps4.md - - SheepShaver: emulators/sheepshaver.md - - Ship of Harkinian: emulators/soh.md - - SimCoupe: emulators/simcp.md - - SimCoupe (standalone): emulators/samcoupe.md - - simple64: emulators/simple64.md - - Singe 2: emulators/singe2.md - - sixtyforce: emulators/sixtyforce.md - - skyemu: emulators/skyemu.md - - Skyline: emulators/skyline.md - - SMS Plus GX: emulators/smsplus.md - - snes9x: emulators/snes9x.md - - Snes9x 2002: emulators/snes9x2002.md - - Snes9x 2005: emulators/snes9x2005.md - - Snes9x 2005 Plus: emulators/snes9x2005_plus.md - - Snes9x 2010: emulators/snes9x2010.md - - Snes9x EX+: emulators/snes9x-explus.md - - solarus: emulators/solarus.md - - Solarus 2.0.0: emulators/solarus-2.0.0.md - - Sonic 1 & 2 (2013) Decompilation: emulators/sonic2013.md - - Sonic 3 A.I.R.: emulators/sonic3-air.md - - Sonic CD Decompilation: emulators/soniccd.md - - Sonic Mania Decompilation: emulators/sonic-mania.md - - Sonic Robo Blast 2: emulators/srb2.md - - Speccy: emulators/speccy.md - - SplitWolf: emulators/splitwolf.md - - SPMP8000Emu: emulators/spmp8000emu.md - - SquirrelJME: emulators/squirreljme.md - - SSF: emulators/ssf.md - - Starboard: emulators/starboard.md - - Starship: emulators/starship.md - - Stella: emulators/stella.md - - Stella 2014: emulators/stella2014.md - - Stella 2023: emulators/stella2023.md - - Sudachi: emulators/sudachi.md - - Sugarbox2: emulators/sugarbox2.md - - Super Mario War: emulators/supermariowar.md - - SUPER ZSNES: emulators/superzsnes.md - - SUPER3: emulators/super3.md - - Supermodel: emulators/supermodel.md - - Supermodel (Sinden fork): emulators/supermodel3.md - - Supermodel Dojo: emulators/supermodel-dojo.md - - Supermodel Legacy: emulators/supermodel-legacy.md - - SuperSnes9x: emulators/supersnes9x.md - - SuperTux: emulators/supertux.md - - SuperTuxKart: emulators/supertuxkart.md - - Suyu: emulators/suyu.md - - Swan.emu: emulators/swan-emu.md - - swanstation: emulators/swanstation.md - - SWF Player: emulators/swf-player.md - - Syobon Action: emulators/syobonaction.md - - TamaLIBretro: emulators/tamalibretro.md - - Taradino: emulators/taradino.md - - TeknoParrot: emulators/teknoparrot.md - - TempGBA: emulators/tempgba.md - - TGB Dual: emulators/tgbdual.md - - The Force Engine: emulators/theforceengine.md - - Theodore: emulators/theodore.md - - TheXTech: emulators/thextech.md - - ti99sim: emulators/ti99sim.md - - ti99sim-sdl1: emulators/ti99sim-sdl1.md - - Tia: emulators/tia.md - - TIC-80: emulators/tic80.md - - touchHLE: emulators/touchhle.md - - TR1X: emulators/tr1x.md - - TR2X: emulators/tr2x.md - - trident: emulators/trident.md - - Triforce: emulators/triforce.md - - tsugaru: emulators/tsugaru.md - - TyrQuake: emulators/tyrquake.md - - UAE4ALL (libretro, Chips-fr): emulators/uae4all-rpi.md - - UAE4All2: emulators/uae4all.md - - uae4arm: emulators/uae4arm.md - - UME 2015: emulators/ume2015.md - - uqm: emulators/uqm.md - - UZDoom: emulators/uzdoom.md - - uzem: emulators/uzem.md + - S: + - SAME CDi: emulators/same_cdi.md + - SameBoy: emulators/sameboy.md + - sameduck: emulators/sameduck.md + - Saturn.emu: emulators/saturn-emu.md + - ScummVM: emulators/scummvm.md + - ScummVM (SDL1): emulators/scummvm-sdl1.md + - SDL2TRS: emulators/sdl2trs.md + - SDLPAL: emulators/sdlpal.md + - sdlpop: emulators/sdlpop.md + - shadps4: emulators/shadps4.md + - SheepShaver: emulators/sheepshaver.md + - Ship of Harkinian: emulators/soh.md + - SimCoupe: emulators/simcp.md + - SimCoupe (standalone): emulators/samcoupe.md + - simple64: emulators/simple64.md + - Singe 2: emulators/singe2.md + - sixtyforce: emulators/sixtyforce.md + - skyemu: emulators/skyemu.md + - Skyline: emulators/skyline.md + - SMS Plus GX: emulators/smsplus.md + - snes9x: emulators/snes9x.md + - Snes9x 2002: emulators/snes9x2002.md + - Snes9x 2005: emulators/snes9x2005.md + - Snes9x 2005 Plus: emulators/snes9x2005_plus.md + - Snes9x 2010: emulators/snes9x2010.md + - Snes9x EX+: emulators/snes9x-explus.md + - solarus: emulators/solarus.md + - Solarus 2.0.0: emulators/solarus-2.0.0.md + - Sonic 1 & 2 (2013) Decompilation: emulators/sonic2013.md + - Sonic 3 A.I.R.: emulators/sonic3-air.md + - Sonic CD Decompilation: emulators/soniccd.md + - Sonic Mania Decompilation: emulators/sonic-mania.md + - Sonic Robo Blast 2: emulators/srb2.md + - Speccy: emulators/speccy.md + - SplitWolf: emulators/splitwolf.md + - SPMP8000Emu: emulators/spmp8000emu.md + - SquirrelJME: emulators/squirreljme.md + - SSF: emulators/ssf.md + - Starboard: emulators/starboard.md + - Starship: emulators/starship.md + - Stella: emulators/stella.md + - Stella 2014: emulators/stella2014.md + - Stella 2023: emulators/stella2023.md + - Sudachi: emulators/sudachi.md + - Sugarbox2: emulators/sugarbox2.md + - Super Mario War: emulators/supermariowar.md + - SUPER ZSNES: emulators/superzsnes.md + - SUPER3: emulators/super3.md + - Supermodel: emulators/supermodel.md + - Supermodel (Sinden fork): emulators/supermodel3.md + - Supermodel Dojo: emulators/supermodel-dojo.md + - Supermodel Legacy: emulators/supermodel-legacy.md + - SuperSnes9x: emulators/supersnes9x.md + - SuperTux: emulators/supertux.md + - SuperTuxKart: emulators/supertuxkart.md + - Suyu: emulators/suyu.md + - Swan.emu: emulators/swan-emu.md + - swanstation: emulators/swanstation.md + - SWF Player: emulators/swf-player.md + - Syobon Action: emulators/syobonaction.md + - T: + - TamaLIBretro: emulators/tamalibretro.md + - Taradino: emulators/taradino.md + - TeknoParrot: emulators/teknoparrot.md + - TempGBA: emulators/tempgba.md + - TGB Dual: emulators/tgbdual.md + - The Force Engine: emulators/theforceengine.md + - Theodore: emulators/theodore.md + - TheXTech: emulators/thextech.md + - ti99sim: emulators/ti99sim.md + - ti99sim-sdl1: emulators/ti99sim-sdl1.md + - Tia: emulators/tia.md + - TIC-80: emulators/tic80.md + - touchHLE: emulators/touchhle.md + - TR1X: emulators/tr1x.md + - TR2X: emulators/tr2x.md + - trident: emulators/trident.md + - Triforce: emulators/triforce.md + - tsugaru: emulators/tsugaru.md + - TyrQuake: emulators/tyrquake.md + - U: + - UAE4ALL (libretro, Chips-fr): emulators/uae4all-rpi.md + - UAE4All2: emulators/uae4all.md + - uae4arm: emulators/uae4arm.md + - UME 2015: emulators/ume2015.md + - uqm: emulators/uqm.md + - UZDoom: emulators/uzdoom.md + - uzem: emulators/uzem.md - V to Z: - - VaporSpec: emulators/vaporspec.md - - VBA-M: emulators/vbam.md - - VBA-Next: emulators/vba_next.md - - vecx: emulators/vecx.md - - veesem: emulators/veesem.md - - VeMUlator: emulators/vemulator.md - - VICE: emulators/vice.md - - VICE x128: emulators/vice_x128.md - - VICE x64: emulators/vice_x64.md - - VICE x64dtv: emulators/vice_x64dtv.md - - VICE x64sc: emulators/vice_x64sc.md - - VICE xcbm2: emulators/vice_xcbm2.md - - VICE xcbm5x0: emulators/vice_xcbm5x0.md - - VICE xpet: emulators/vice_xpet.md - - VICE xplus4: emulators/vice_xplus4.md - - VICE xscpu64: emulators/vice_xscpu64.md - - VICE xvic: emulators/vice_xvic.md - - Vircon32: emulators/vircon32.md - - Virtual Jaguar: emulators/virtualjaguar.md - - Virtual Virtual Boy: emulators/virtual-virtual-boy.md - - VirtualXT: emulators/virtualxt.md - - visual-pinball: emulators/visual-pinball.md - - Vita3K: emulators/vita3k.md - - vitaQuakeII: emulators/vitaquake2.md - - vitaQuakeIII: emulators/vitaquake3.md - - vitavoyager: emulators/vitavoyager.md - - vkQuake: emulators/vkquake.md - - vkQuake2: emulators/vkquake2.md - - vkQuake3: emulators/vkquake3.md - - VVVVVV: emulators/vvvvvv.md - - WinUAE: emulators/winuae.md - - Wolf4SDL: emulators/wolf4sdl.md - - WQXEmu: emulators/wqxemu.md - - X Millennium: emulators/x1.md - - X1 BOX: emulators/x1-box.md - - x16emu: emulators/x16emu.md - - x64sdl: emulators/x64sdl.md - - Xash3D FWGS: emulators/xash3d_fwgs.md - - xemu: emulators/xemu.md - - xemu Chihiro: emulators/xemu-chihiro.md - - XenDroid: emulators/xendroid.md - - Xenia Canary: emulators/xenia.md - - Xenia Edge: emulators/xenia-edge.md - - XeniOS: emulators/xenios.md - - xm6pro68k: emulators/xm6pro68k.md - - xm6typeg: emulators/xm6typeg.md - - XM7: emulators/xm7.md - - XRick: emulators/xrick.md - - XRoar: emulators/xroar.md - - yabasanshiro: emulators/yabasanshiro.md - - yabause: emulators/yabause.md - - Yamagi Quake II: emulators/yquake2.md - - yaps2: emulators/yaps2.md - - ymir: emulators/ymir.md - - Yuzu: emulators/yuzu.md - - Zelda Classic 2.55 Prototype: emulators/zc255.md - - Zelda Classic v2.10: emulators/zc210.md - - ZEsarUX: emulators/zesarux.md - - ZiNc: emulators/zinc.md - - ZQuest Classic: emulators/zquestclassic.md + - V: + - VaporSpec: emulators/vaporspec.md + - VBA-M: emulators/vbam.md + - VBA-Next: emulators/vba_next.md + - vecx: emulators/vecx.md + - veesem: emulators/veesem.md + - VeMUlator: emulators/vemulator.md + - VICE: emulators/vice.md + - VICE x128: emulators/vice_x128.md + - VICE x64: emulators/vice_x64.md + - VICE x64dtv: emulators/vice_x64dtv.md + - VICE x64sc: emulators/vice_x64sc.md + - VICE xcbm2: emulators/vice_xcbm2.md + - VICE xcbm5x0: emulators/vice_xcbm5x0.md + - VICE xpet: emulators/vice_xpet.md + - VICE xplus4: emulators/vice_xplus4.md + - VICE xscpu64: emulators/vice_xscpu64.md + - VICE xvic: emulators/vice_xvic.md + - Vircon32: emulators/vircon32.md + - Virtual Jaguar: emulators/virtualjaguar.md + - Virtual Virtual Boy: emulators/virtual-virtual-boy.md + - VirtualXT: emulators/virtualxt.md + - visual-pinball: emulators/visual-pinball.md + - Vita3K: emulators/vita3k.md + - vitaQuakeII: emulators/vitaquake2.md + - vitaQuakeIII: emulators/vitaquake3.md + - vitavoyager: emulators/vitavoyager.md + - vkQuake: emulators/vkquake.md + - vkQuake2: emulators/vkquake2.md + - vkQuake3: emulators/vkquake3.md + - VVVVVV: emulators/vvvvvv.md + - W: + - WinUAE: emulators/winuae.md + - Wolf4SDL: emulators/wolf4sdl.md + - WQXEmu: emulators/wqxemu.md + - X: + - X Millennium: emulators/x1.md + - X1 BOX: emulators/x1-box.md + - x16emu: emulators/x16emu.md + - x64sdl: emulators/x64sdl.md + - Xash3D FWGS: emulators/xash3d_fwgs.md + - xemu: emulators/xemu.md + - xemu Chihiro: emulators/xemu-chihiro.md + - XenDroid: emulators/xendroid.md + - Xenia Canary: emulators/xenia.md + - Xenia Edge: emulators/xenia-edge.md + - XeniOS: emulators/xenios.md + - xm6pro68k: emulators/xm6pro68k.md + - xm6typeg: emulators/xm6typeg.md + - XM7: emulators/xm7.md + - XRick: emulators/xrick.md + - XRoar: emulators/xroar.md + - Y: + - yabasanshiro: emulators/yabasanshiro.md + - yabause: emulators/yabause.md + - Yamagi Quake II: emulators/yquake2.md + - yaps2: emulators/yaps2.md + - ymir: emulators/ymir.md + - Yuzu: emulators/yuzu.md + - Z: + - Zelda Classic 2.55 Prototype: emulators/zc255.md + - Zelda Classic v2.10: emulators/zc210.md + - ZEsarUX: emulators/zesarux.md + - ZiNc: emulators/zinc.md + - ZQuest Classic: emulators/zquestclassic.md - All installer options: wiki/installer.md - Reference: - wiki/index.md @@ -835,7 +859,7 @@ nav: - Dump provenance: provenance.md - Updated platform files: exports.md - Data and API: data.md - - Find a file by its SHA1: files/index.md + - Find a file: files/index.md - Data model: wiki/data-model.md - Architecture: wiki/architecture.md - Build your own packs: wiki/advanced-usage.md diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 18bf24ff..dcd422b2 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -27,7 +27,8 @@ "emulators", "systems", "verification-and-coverage-gaps", - "file-locations" + "file-locations", + "file-lookup" ] }, "count": { @@ -332,6 +333,66 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "file-lookup" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "value", + "algorithm", + "layer", + "name", + "url" + ], + "properties": { + "value": { + "type": "string", + "pattern": "^[0-9a-f]+$" + }, + "algorithm": { + "enum": [ + "sha1", + "md5", + "crc32", + "sha256" + ] + }, + "layer": { + "enum": [ + "collection", + "data-directory", + "platform", + "emulator" + ] + }, + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "owner": { + "type": "string" + } + } + } + } + } + } } ] } diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index 8b8c2b66..7af1656c 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open. from __future__ import annotations import sys +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path @@ -136,6 +137,81 @@ class ReadEvidence: return None +# The order ReadEvidence.proof tries: a reader's proof is the first that holds. +_PROOF_ORDER = ("path", "launcher", "name", "hash", "directory", "data_directory") + + +class ProofIndex: + """ReadEvidence.proof for many readers at once, by lookup. + + Asking every core in turn cost one proof per core and entry: 3.8 million + calls to categorize 7,430 entries. Each key a reader can match is indexed + once, and an entry's own keys find the readers it concerns. The proof of + each reader is the one ReadEvidence.proof returns. + """ + + def __init__(self, readers: list[ReadEvidence]) -> None: + self._index: dict[str, dict[str, set[int]]] = { + kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member") + } + for position, evidence in enumerate(readers): + for kind, keys in ( + ("tail", evidence.tails), ("launched", evidence.launched), + ("name", evidence.names), ("hash", evidence.hashes), + ("dir", evidence.directories), ("tree", evidence.trees), + ("member", evidence.tree_members), + ): + for key in keys: + self._index[kind].setdefault(key, set()).add(position) + # A cut md5 names any hash it is the start of. + for value in evidence.hashes: + for length in range(_CUT_MD5, _FULL_MD5): + if len(value) >= length: + self._index["cut"].setdefault(value[:length], set()).add(position) + + def _hits(self, kind: str, keys: Iterable[str]) -> set[int]: + table = self._index[kind] + found: set[int] = set() + for key in keys: + found |= table.get(key, set()) + return found + + def proofs(self, entry: dict, destination: str) -> dict[int, str]: + """Reader position -> its proof, for the readers that have one.""" + names = { + str(n).casefold() + for n in (entry.get("name"), *(entry.get("aliases") or [])) + if n + } + tails = _tails(destination) + last = {tails[-1]} if tails else set() + hashes = { + v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f) + } + cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5} + # "/" + d in "/" + path: d is a run of whole segments ending in "/", + # so it is a run of the path's segments before the last one. + parts = tails[0].split("/") if tails else [] + runs = { + "/".join(parts[i:j]) + "/" + for i in range(len(parts)) for j in range(i + 1, len(parts)) + } + found = { + "path": self._hits("tail", tails[:-1]), + "launcher": self._hits("launched", names | last), + "name": self._hits("name", names | last), + "hash": self._hits("hash", hashes) | self._hits("cut", cut), + "directory": self._hits("dir", runs), + "data_directory": self._hits("tree", runs) + | self._index["member"].get(tails[0] if tails else "", set()), + } + result: dict[int, str] = {} + for kind in _PROOF_ORDER: + for position in found[kind]: + result.setdefault(position, kind) + return result + + def _loads_romsets(profile: dict) -> bool: """A core that opens ROM sets by name: MAME pulls a device's set from device_t::searchpath(), and no profile lists every device set.""" @@ -275,6 +351,8 @@ class EntryCategories: self._readers.append((kind, key, evidence)) self._launchers = ReadEvidence() self._launchers.add_launchers(config.get("launchers") or []) + self._proofs = ProofIndex([evidence for _kind, _key, evidence in self._readers] + + [self._launchers]) self._by_system: dict[str, list[str]] = {} for key in sorted(examined): for sid in profiles[key].get("systems") or []: @@ -313,14 +391,14 @@ class EntryCategories: def explain(self, entry: dict, destination: str, system: str) -> dict: """The category of an entry and what establishes it.""" readers: dict[str, list[dict]] = {BIOS: [], GAME_DATA: []} - for kind, key, evidence in self._readers: - proof = evidence.proof(entry, destination) - if proof is not None: - readers[kind].append({ - "emulator": key, "proof": proof, - "entries": self._matching(key, kind, entry), - }) - if self._launchers.proof(entry, destination) is not None: + proofs = self._proofs.proofs(entry, destination) + for position in sorted(p for p in proofs if p < len(self._readers)): + kind, key, _evidence = self._readers[position] + readers[kind].append({ + "emulator": key, "proof": proofs[position], + "entries": self._matching(key, kind, entry), + }) + if len(self._readers) in proofs: readers[BIOS].append({"emulator": "launcher", "proof": "launcher", "entries": []}) if system in self._system_runs: system_cores, unresolved = self._system_runs[system] diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 1a8ab504..519d4202 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -111,6 +111,105 @@ def compute_coverage( } +_REPO = Path(__file__).resolve().parent.parent +# The site and the README read the same coverage: the first generator to run +# stores it here and the second reads it back, instead of verifying every +# platform twice (about a minute of the site build). +COVERAGE_CACHE_DIR = _REPO / "tmp" / "coverage-cache" +_COVERAGE_CACHE_FORMAT = "1" + + +def _coverage_inputs(db_path: Path, platforms_dir: str, emulators_dir: str) -> str: + """A digest of everything coverage is computed from. + + The database, the platform and emulator files, the code, the install + manifests, the data directory revisions and the state of bios/ on disk: + a change to any of them computes the coverage again. + """ + import hashlib + + digest = hashlib.sha256(_COVERAGE_CACHE_FORMAT.encode()) + + def add_file(path: Path) -> None: + digest.update(str(path).encode() + b"\0") + digest.update(path.read_bytes() if path.is_file() else b"") + + add_file(db_path.resolve()) + for root, pattern in ((Path(platforms_dir), "*"), (Path(emulators_dir), "*.yml"), + (_REPO / "scripts", "*.py"), (_REPO / "install", "*.json"), + (_REPO / "recipes", "*.json")): + for path in sorted(p for p in root.resolve().rglob(pattern) if p.is_file()): + add_file(path) + for path in (_REPO / ".gitignore", _REPO / "_mame_clones.json", + _REPO / "data" / ".versions.json"): + add_file(path) + bios = _REPO / "bios" + for path in sorted(bios.rglob("*")): + if path.is_file(): + stat = path.stat() + digest.update(f"{path.relative_to(bios)}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode()) + return digest.hexdigest() + + +def platform_coverages( + platform_names: list[str], + platforms_dir: str, + db: dict, + data_registry: dict | None = None, + emulators_dir: str = _EMULATORS_DIR, + db_path: str = "database.json", + report: Callable[[str, dict], None] | None = None, +) -> dict[str, dict]: + """compute_coverage for each platform, read from the cache when its + inputs have not changed since a generator last stored it. + + A platform with no YAML is left out, as compute_coverage raises for it. + ``report`` is called with each platform and its coverage. + """ + # pickle, not JSON: the coverage holds sets and enums that JSON would + # turn into lists and strings. The file is written by this function into + # the repository's own tmp/, never downloaded. + import pickle + + key = _coverage_inputs(Path(db_path), platforms_dir, emulators_dir) + names = sorted(platform_names) + cache = COVERAGE_CACHE_DIR / f"{key}.pickle" + coverages: dict[str, dict] | None = None + if cache.is_file(): + try: + with cache.open("rb") as handle: + stored = pickle.load(handle) + except (OSError, EOFError, pickle.UnpicklingError) as exc: + print(f" coverage cache unreadable, computing again: {exc}", file=sys.stderr) + else: + if stored.get("platforms") == names: + coverages = stored["coverages"] + print(f" coverage read from {cache.relative_to(_REPO)}") + if coverages is None: + coverages = {} + for name in names: + try: + coverages[name] = compute_coverage( + name, platforms_dir, db, data_registry, emulators_dir=emulators_dir, + ) + except FileNotFoundError as exc: + print(f" {name}: skipped ({exc})", file=sys.stderr) + COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True) + for stale in COVERAGE_CACHE_DIR.glob("*.pickle"): + stale.unlink() + scratch = cache.with_suffix(f".{os.getpid()}.tmp") + with scratch.open("wb") as handle: + pickle.dump({"platforms": names, "coverages": coverages}, handle, + protocol=pickle.HIGHEST_PROTOCOL) + scratch.replace(cache) + ordered = {name: coverages[name] for name in platform_names if name in coverages} + if report: + for name in names: + if name in ordered: + report(name, ordered[name]) + return ordered + + def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]: """Files and bytes of the pack the latest release serves for a platform. @@ -347,7 +446,7 @@ def _catalog_matched_line(db: dict) -> list[str]: ] -def generate_readme(db: dict, platforms_dir: str) -> str: +def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json") -> str: total_files = db.get("total_files", 0) total_size = db.get("total_size", 0) ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -358,14 +457,8 @@ def generate_readme(db: dict, platforms_dir: str) -> str: data_registry = load_data_dir_registry(platforms_dir) - coverages = {} - for name in platform_names: - try: - coverages[name] = compute_coverage( - name, platforms_dir, db, data_registry - ) - except FileNotFoundError: - pass + coverages = platform_coverages(platform_names, platforms_dir, db, data_registry, + db_path=db_path) profiles = unique_emulator_profiles( load_emulator_profiles("emulators", skip_aliases=False) @@ -853,7 +946,7 @@ def main() -> None: # published counts taken from half of it. with contextlib.ExitStack() as held: hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) - readme = generate_readme(db, args.platforms_dir) + readme = generate_readme(db, args.platforms_dir, args.db) status = "Generated" if write_if_changed("README.md", readme) else "Unchanged" print(f"{status} ./README.md") diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 6266e04c..310a49cc 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -65,6 +65,7 @@ from nativemode import digest_algorithm, reads_file_contents yaml = require_yaml() from generate_readme import ( compute_coverage, + platform_coverages, contributing_text, download_links, extract_notes, @@ -79,6 +80,7 @@ from siteguides import ( DEVICE_GROUPS, GROUP_ANCHORS, GUIDES, + Guide, INSTALL_BASE, install_command, platform_args, @@ -1130,13 +1132,17 @@ def generate_data_exports( gap_rows = _gap_export_rows(coverages, gap_report) cross_rows = _cross_reference_export_rows(coverages, profiles) + members = data_directory_members() + locations = file_location_items(db, profiles, members) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), "gaps.json": ("verification-and-coverage-gaps", gap_rows), - "file-locations.json": ("file-locations", file_location_items( - db, profiles, data_directory_members())), + "file-locations.json": ("file-locations", locations), } + write_lookup_shards( + api, file_lookup_records(db, coverages, profiles, locations, members), generated_at, + ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, @@ -1338,6 +1344,13 @@ def generate_data_page(exports: list[dict]) -> str: "platform of `platforms.json`, and `api/v1/systems/.json` the files " "of one system page by console. Each platform and system page links its own.", "", + "One hash without the whole list: `api/v1/lookup/.json`, where `` " + "is the first two characters of the hash in lowercase, holds every SHA1, " + "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " + "(the collection, a data directory, a platform's list or an emulator " + "profile), the file `name` and the `url` of the page that shows it. " + "[Find a file](files/index.md) reads it.", + "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", @@ -1418,6 +1431,8 @@ def _browser_title(relative: Path, title: str) -> str: } if key in index_titles: return index_titles[key] + if key in PART_PAGES: + return title if key.startswith("emulators/"): return f"{title} emulator firmware" if key.startswith("systems/"): @@ -1462,6 +1477,9 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: } if key in specific: return specific[key] + if key in PART_PAGES: + return (f"{title}: every file with its hashes, size, source references " + "and who reads it.") if key.startswith("platforms/") and relative.stem != "index": return (f"{title}: where the BIOS files go, the command that installs them, the pack, " "and every file by system with its hashes.") @@ -1498,9 +1516,13 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: # Structured data a page generator knows beyond its text, by page path # relative to docs/: read by decorate_markdown_pages. PAGE_DATA: dict[str, dict] = {} +# Pages that hold part of a page too long for one (a console of a split +# manufacturer, a run of an emulator's cards), by path relative to docs/: +# their H1 already says what they are. +PART_PAGES: set[str] = set() -SYSTEM_PARTS_IN_FULL = 1000 +SYSTEM_PARTS_IN_FULL = 250 def collection_dat(db: dict, generated_at: str) -> str: @@ -1589,8 +1611,9 @@ def system_structured_data(slug: str, consoles: dict[str, list]) -> dict: part["citation"] = citations parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: - # game-engines would carry 2.6 MB of JSON-LD: past the cap a page - # lists its catalogue-verified files and points at the full table. + # A page of cards carried up to 2.6 MB of JSON-LD (game-engines), a + # quarter of its weight: past the cap a page lists its + # catalogue-verified files and points at the full table. return { "hasPart": [part for part in parts if part.get("citation")], "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"), @@ -1812,6 +1835,11 @@ def emulator_structured_data(profile: dict) -> dict: if members: part["hasPart"] = members parts.append(part) + if len(parts) > SYSTEM_PARTS_IN_FULL: + # MAME's 1,564 entries and their members were 614 KB of JSON-LD: past + # the cap the members are left to the profile's JSON. + for part in parts: + part.pop("hasPart", None) if parts: data["hasPart"] = parts return data @@ -2032,14 +2060,23 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st return "\n".join(lines) + "\n" +# A platform system whose file list passes this much Markdown gets a page +# of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page. +PLATFORM_SECTION_BUDGET = 200_000 + + def _render_platform_systems( - by_system: dict, config_files: dict, emulator_files: dict + by_system: dict, config_files: dict, emulator_files: dict, + platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None, ) -> list[str]: """One collapsible block per system, listing every file it needs. Split out of generate_platform_page: the block is most of that function's branching, since each file renders a different set of hashes, sizes and provenance depending on what the platform declares. + With ``platform`` (its key and name) and ``moved``, a list longer than + PLATFORM_SECTION_BUDGET goes to a page of its own, written into + ``moved`` by stem, and the system's heading stays with a link to it. """ lines: list[str] = [] # Per-system detail sections (collapsible for large platforms) @@ -2059,6 +2096,7 @@ def _render_platform_systems( # emulator cards link here, and the site search splits a page at its # headings. Folded into blocks under one heading, every hash pasted # in the search led to the systems overview of a 100-block page. + block_start = len(lines) + 2 if use_collapsible: status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") lines.append(f"### {sys_link} {{ #{anchor} }}") @@ -2141,7 +2179,29 @@ def _render_platform_systems( if f.get("category_evidence"): lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}") - lines.append("") + block = lines[block_start:] + if moved is not None and platform and sum(len(line) + 1 for line in block) > PLATFORM_SECTION_BUDGET: + key, display = platform + stem = f"{key}-{anchor}" + body = block[2:] if use_collapsible else block + moved[stem] = "\n".join([ + f"# {display}: {sys_id}", "", + f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " + f"are listed there. Each file below is as {display} declares it, with " + "its status in the pack.", + "", "## Files", "", + *search_excluded( + [line[len(pad):] if line.startswith(pad) else line for line in body]), + "", f"*Generated on {_timestamp()}*", + ]) + "\n" + lines[block_start:] = [ + f"{_count(total, 'file')}, {ok_count} verified: listed on " + f"[a page of their own]({stem}.md).", "", + ] + continue + # The section's heading stays searchable, its file list with every + # hash is the lookup's: retrodeck's arcade list was 495 KB of index. + lines[block_start:] = search_excluded(lines[block_start:]) return lines @@ -2371,9 +2431,13 @@ def generate_platform_page( cov: dict, registry: dict | None = None, emulator_files: dict | None = None, + moved: dict[str, str] | None = None, ) -> str: """A guide first (folder, command, pack, check), then every file the - platform's list and its emulators need, under a reference heading.""" + platform's list and its emulators need, under a reference heading. + + ``moved`` collects the pages of systems whose list is too long for it. + """ registry = registry or {} config = cov["config"] display = cov["platform"] @@ -2470,7 +2534,8 @@ def generate_platform_page( "", "", ] - lines.extend([*band, "| | |", "|---|---|"]) + lines.extend([*band, '
', "", + "| Detail | Value |", "|---|---|"]) if hash_type: lines.append(f"| Hash type | {hash_type} |") if version: @@ -2504,6 +2569,7 @@ def generate_platform_page( f"| Contributed by | [@{username}](https://github.com/{username}):" f" {contribution}{pr_link} |" ) + lines.extend(["", "
"]) if name not in GUIDES: detect = (registry.get(name, {}).get("install") or {}).get("detect") or [] on_card = any(rule.get("method") == "card_marker" for rule in detect) @@ -2561,7 +2627,8 @@ def generate_platform_page( lines.append("") lines.extend( - _render_platform_systems(by_system, config_files, emulator_files) + _render_platform_systems(by_system, config_files, emulator_files, + (name, display), moved) ) lines.append(f"*Generated on {_timestamp()}*") @@ -2572,28 +2639,44 @@ def generate_platform_page( def generate_files_page() -> str: - """The content address of every held file: files/?sha1=. + """Find a file by any of its hashes: files/?hash=. - site.js reads api/v1/file-locations.json and goes to the card. Without - a sha1, or with one the collection does not hold, the page says so. + site.js reads the one shard of api/v1/lookup/ the value names and lists + every page that cites it. files/?sha1= is the card's permanent + address and opens the card directly. """ return "\n".join([ - "# Find a file by its SHA1", + "# Find a file", "", - "A file's card lives on its manufacturer's page, and a file refiled " - "between manufacturers moves with it. `files/?sha1=` is the " - "address that does not move: it opens the card wherever it is now. " - "Each card's **permalink** is that address.", + "Paste a hash from clrmamepro, RomVault, an emulator log or a platform's " + "BIOS list. The page lists the file in the collection that has it, the " + "platforms that declare it and the emulators that check for it.", "", '
', - ' ', - ' ', - '', + ' ', + ' ', + '', "
", "", '

', "", - "The index behind it is [`api/v1/file-locations.json`](../api/v1/file-locations.json).", + "To look a file up by its name, use the search at the top of the page.", + "", + "## Permanent links", + "", + "A file's card lives on its manufacturer's page, and a file refiled " + "between manufacturers moves with it. `files/?sha1=` is the address " + "that does not move: it opens the card wherever it is now. Each card's " + "**permalink** is that address.", + "", + "## The index behind it", + "", + "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " + "first two characters of the hash, about 20 KB compressed. " + "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " + "card by SHA1 in one file.", "", ]) @@ -2670,8 +2753,8 @@ def file_location_items( parts = entry.get("path", "").split("/") if len(parts) < 3 or parts[0] != "bios": continue - slug = parts[1].lower().replace(" ", "-") - items.append({"sha1": sha1, "url": f"systems/{slug}/#sha1-{sha1[:12]}"}) + page = card_page(entry.get("path", ""), sha1) + items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"}) # A data directory's files are listed on the page of an emulator reading # the tree: 2,534 hashes the packs ship were found nowhere on the site. owners = _data_member_owners(profiles or {}) @@ -2688,6 +2771,99 @@ def file_location_items( return sorted(items, key=lambda item: item["sha1"]) +LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files + + +def file_lookup_records( + db: dict, + coverages: dict, + profiles: dict, + locations: list[dict], + members: dict | None = None, +) -> list[dict]: + """Every hash the site knows, one record per value, for files/?hash=. + + The collection's own files lead to their card, a platform's declared + value to the system section of its page, a profile's to the entry's + card. The site search indexed these values as text, 4 MB of the 21 MB + it made every visitor download: the lookup reads one shard instead. + """ + files = db.get("files", {}) + member_names = { + str(info.get("sha1", "")): member.rsplit("/", 1)[-1] + for tree in (members or {}).values() for member, info in sorted(tree.items()) + } + records: list[dict] = [] + for item in locations: + sha1 = item["sha1"] + entry = files.get(sha1) + if entry is None: + # A data directory member: the manifests record its sha1 only. + records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", + "name": member_names.get(sha1, ""), "sha1": sha1, + "url": item["url"]}) + continue + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "collection", + "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], + }) + for key, coverage in sorted(coverages.items()): + for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): + url = f"platforms/{key}/#{system_id.replace(' ', '-')}" + for entry in system.get("files") or []: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "platform", + "owner": key, "name": str(entry.get("name", "")), "url": url, + }) + for key, profile in sorted(profiles.items()): + if profile.get("type") in ("alias", "test"): + continue + entries = profile.get("files") or [] + for entry, anchor in zip(entries, emulator_file_anchors(entries)): + url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}" + members = [(str(entry.get("name", "")), entry)] + [ + (str(member["name"]), member) for member in entry.get("contents") or [] + if isinstance(member, dict) and member.get("name") + ] + for name, holder in members: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(holder.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "emulator", + "owner": key, "name": name, "url": url, + }) + unique = {json.dumps(r, sort_keys=True): r for r in records} + return sorted(unique.values(), key=lambda r: ( + r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) + + +def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int: + """One envelope per leading pair of hex characters, as the lookup asks. + + A value that is not hexadecimal (a typo in a profile) has no shard the + lookup would read, and is left out. Returns the number of shards. + """ + shards: dict[str, list[dict]] = {} + for record in records: + if re.fullmatch(r"[0-9a-f]{8,}", record["value"]): + shards.setdefault(record["value"][:LOOKUP_PREFIX], []).append(record) + folder = api / "lookup" + folder.mkdir(parents=True, exist_ok=True) + for prefix, items in sorted(shards.items()): + write_if_changed(str(folder / f"{prefix}.json"), json.dumps({ + "schema_version": 1, + "generated_at": generated_at, + "kind": "file-lookup", + "count": len(items), + "items": items, + }, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n") + return len(shards) + + def collection_paths(db: dict) -> dict[str, set[str]]: """Every path under bios/ that holds each file, by sha1. @@ -3083,8 +3259,9 @@ def _emulator_links(sha1: str, consumers: FileConsumers) -> list[str]: index = entries.get(emulator) anchor = (_emulator_card_anchor(emulator, files_of[emulator], index) if index is not None and emulator in files_of else "") + page = emulator_card_page(emulator, anchor) if anchor else emulator fragment = f"#{anchor}" if anchor else "" - links.append(f"[{emulator}](../emulators/{emulator}.md{fragment})") + links.append(f"[{emulator}](../emulators/{page}.md{fragment})") return links @@ -3101,9 +3278,14 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> def _system_file_card( - f: dict, consumers: FileConsumers, anchored: set[str] | None = None + f: dict, consumers: FileConsumers, anchored: set[str] | None = None, + searchable: bool = True, ) -> list[str]: - """One file of a system page: hashes, who reads it, its provenance.""" + """One file of a system page: hashes, who reads it, its provenance. + + ``searchable`` false keeps even the name out of the site search: the + asset trees of game engines hold thousands of 0.ogg and tile.png. + """ name = f["name"] sha1_full = f.get("sha1", "unknown") md5_full = f.get("md5", "unknown") @@ -3129,7 +3311,7 @@ def _system_file_card( ) if anchor else "" lines.append(_card_title( f'{_code(name)} {size}{permalink}', - anchor, + anchor, searchable, )) lines.append("") # Every hash in full: the site search indexes text, not a title @@ -3160,17 +3342,68 @@ def _system_file_card( # looking up scph5501.bin found every platform but RetroBat, whose # pack installs it for six cores. lines.append(f"- Also installed by: {', '.join(installed)}") - lines[2:] = _search_skipped_apart(lines[2:]) + # The title stays searchable: the file name is what a reader types. + lines[4:] = search_excluded(lines[4:]) lines.extend(["", "", ""]) return lines +SEARCH_TABLE_BYTES = 3000 + + +def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> str: + """Each table longer than ``threshold`` bytes, wrapped out of the search. + + A search result shows the start of a section: a 600-row table as one + snippet helps nobody, and the gaps, provenance and cross-reference + tables were 900 KB of the index. The headings above them stay. + """ + lines = markdown.split("\n") + out: list[str] = [] + index = 0 + while index < len(lines): + match = re.match(r"( *)\|", lines[index]) + if not match: + out.append(lines[index]) + index += 1 + continue + indent = match.group(1) + end = index + while end < len(lines) and re.match(rf"{indent}\|", lines[end]): + end += 1 + table = lines[index:end] + if sum(len(line) for line in table) > threshold: + out += ["", f"{indent}
", "", *table, + "", f"{indent}
", ""] + else: + out += table + index = end + return "\n".join(out) + + +def search_excluded(lines: list[str]) -> list[str]: + """Lines the site search leaves out, as a block of their own. + + The browser downloads and indexes the whole search index before the + first result: hashes, link lists and install commands repeated on 600 + pages made it 21 MB. Hashes are found by files/?hash=, names and prose + stay searchable. + + A section, not a div: the search plugin compares elements by tag, so + the first div closed inside an excluded div (a code block's) ended the + exclusion and the rest of the block was indexed. + """ + return ["", "
", "", *lines, "", "
", ""] + + # Card lines the site search leaves out. They list where a file goes and # who cites it, links to other pages that index the same names, and they # were 6.6 of the 24.7 MB the browser downloads and indexes before the first # result. The name, the hashes, the entry's path and the notes stay # searchable. SEARCH_SKIPPED = ( + "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:", + "- Validation", "- Priority:", "- Contents (", "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:", "- Collection path:", "- Collection copy:", "- Collection copies:", "- Installed by the platform's own package on:", @@ -3195,16 +3428,20 @@ def _search_skipped_apart(lines: list[str]) -> list[str]: return out -def _card_title(text: str, anchor: str) -> str: +def _card_title(text: str, anchor: str, searchable: bool = True) -> str: """A file card's heading, so the site search lands on the card. The search splits a page at its headings: a card titled in bold sent a - pasted md5 to the top of a page of a hundred cards. Level 4 stays out - of the table of contents (toc_depth 3), and a card without an anchor - keeps the id the toc gives it. + pasted md5 to the top of a page of a hundred cards. Level 3, under + the console or the Files heading: a level 4 after a level 2 skipped + one for screen readers on 551 pages. The table of contents of a page + of cards stops above it (mkdocs_hooks.py), and a card without an + anchor keeps the id the toc gives it. """ attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" - return f"#### {text} {{ {attributes} }}" + if not searchable: + attributes += " data-search-exclude" + return f"### {text} {{ {attributes} }}" def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[str, str]]]]: @@ -3236,6 +3473,197 @@ def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[st return choices +# A manufacturer page above this much Markdown becomes a summary with one +# page per console, and a console above it is cut into parts. 500 KB of +# Markdown renders to about 750 KB of HTML: game-engines was one 10.5 MB +# page that took 21 s of main-thread work on a phone. +SYSTEM_PAGE_BUDGET = 500_000 + +# A system card: whether it is a variant, its lines, its sha1. +Card = tuple[bool, list[str], str] + + +def _console_cards( + manufacturer: str, + files: list[dict], + consumers: FileConsumers, + anchored: set[str], +) -> list[Card]: + """Each card of one console, primaries by name then variants, with + whether it is a variant.""" + searchable = manufacturer not in GAME_DATA_TOPS + main_files = [f for f in files if "/.variants/" not in f["path"]] + variant_files = [f for f in files if "/.variants/" in f["path"]] + cards = [(False, _system_file_card(f, consumers, anchored, searchable), str(f.get("sha1", ""))) + for f in sorted(main_files, key=lambda x: x["name"])] + cards += [(True, _system_file_card(v, consumers, anchored, searchable), str(v.get("sha1", ""))) + for v in sorted(variant_files, key=lambda x: (x["name"], x["path"]))] + return cards + + +def _cards_markdown(cards: list[Card]) -> list[str]: + """Primaries, then the variants in a closed block.""" + lines = [line for variant, card, _sha1 in cards if not variant for line in card] + variants = [card for variant, card, _sha1 in cards if variant] + if variants: + # A variant gets the same card as a primary: the revision a + # platform pins is the one a reader comes to find, and showed as + # a name and a truncated MD5, nothing said who pins it. + lines.append(f'??? note "Variants ({len(variants)})"') + lines.append("") + for card in variants: + lines.extend(f" {line}" if line else "" for line in card) + lines.append("") + return lines + + +def _parts(cards: list[Card], budget: int) -> list[list[Card]]: + """Cards in runs of at most ``budget`` bytes of Markdown, in order.""" + parts: list[list[Card]] = [[]] + size = 0 + for card in cards: + weight = sum(len(line) + 1 for line in card[1]) + if parts[-1] and size + weight > budget: + parts.append([]) + size = 0 + parts[-1].append(card) + size += weight + return parts + + +_NEXT_FROM_SYSTEMS = [ + "[Get your files](../which-pack.md), for the setup you play on", + "[When a game does not start](../wiki/troubleshooting.md)", + "[Every emulator](../emulators/index.md), and the files each one reads", +] + + +def system_pages( + manufacturer: str, + consoles: dict[str, list], + consumers: FileConsumers, + installs: dict[str, list[tuple[str, str]]] | None = None, + budget: int = SYSTEM_PAGE_BUDGET, +) -> dict[str, tuple[str, dict[str, list]]]: + """A manufacturer's pages by stem under systems/: one page, or a summary + and a page per console when the whole would pass ``budget``. Each page + comes with the files whose cards it holds, by console. + + Every console keeps its heading on the summary, so a link to + systems//# still lands, and site.js sends a card + anchor the page no longer holds to the card's new place. The pages + each card and console went to are recorded for every other link + (siterender.record_console_pages). + """ + slug = _slug(manufacturer) + json_link = (f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json)") + whole_anchors: set[str] = set() + sections = { + console: _console_cards(manufacturer, consoles[console], consumers, whole_anchors) + for console in sorted(consoles) + } + lines = [ + f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as {json_link}.", + "", + ] + for console, cards in sections.items(): + lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}") + lines.append("") + lines.extend(_console_install_block(console, (installs or {}).get(console, []))) + lines.extend(_cards_markdown(cards)) + lines.append("") + lines.extend(_next_block(_NEXT_FROM_SYSTEMS)) + lines.append(f'
Generated on {_timestamp()}.
') + whole = "\n".join(lines) + "\n" + if len(whole) <= budget: + return {slug: (whole, consoles)} + + # The heaviest sections move to pages of their own until what stays on + # the manufacturer's page is half the budget: a section of one file + # keeps its place, and a reader does not open 91 pages to browse Other. + weights = { + console: sum(len(line) + 1 for _variant, card, _sha1 in cards for line in card) + for console, cards in sections.items() + } + moved: set[str] = set() + for console in sorted(weights, key=lambda c: (-weights[c], c)): + if sum(w for c, w in weights.items() if c not in moved) <= budget // 2: + break + moved.add(console) + + pages: dict[str, tuple[str, dict[str, list]]] = {} + summary = [ + f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as {json_link}. " + f"The largest sections have a page of their own, linked from their heading.", + "", + ] + summary_anchors: set[str] = set() + for console in sorted(consoles): + summary.append(f"## {system_icon_markdown(manufacturer, console)}{console}") + summary.append("") + if console not in moved: + summary.extend(_console_install_block(console, (installs or {}).get(console, []))) + summary.extend(_cards_markdown( + _console_cards(manufacturer, consoles[console], consumers, summary_anchors))) + summary.append("") + continue + # Cards are rendered again for their own page: the anchors a page + # holds are its own. + cards = _console_cards(manufacturer, consoles[console], consumers, set()) + parts = _parts(cards, budget) + base = f"{slug}-{_slugify_anchor(console) or 'files'}" + stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))] + record_console_pages(slug, console, stems, [ + [sha1 for _variant, _card, sha1 in part] for part in parts + ]) + title = f"{manufacturer}: {console}" + count = _count(len(consoles[console]), "file") + if len(stems) == 1: + summary.append(f"{count}, on [their own page]({stems[0]}.md).") + else: + summary.append(f"{count}, on {len(stems)} pages by file name:") + summary.append("") + for index, (stem, part) in enumerate(zip(stems, parts)): + summary.append(f"- [Part {index + 1}]({stem}.md): {_part_range(part)}") + summary.append("") + for index, (stem, part) in enumerate(zip(stems, parts)): + heading = title if len(stems) == 1 else f"{title}, part {index + 1} of {len(stems)}" + lines = [ + f"# {heading}", "", + f"Part of [{manufacturer}]({slug}.md), in [Systems](index.md). " + f"Every hash of these files is also published as {json_link}.", + "", + ] + if len(stems) > 1: + others = ", ".join( + f"[part {i + 1}]({other}.md)" for i, other in enumerate(stems) if i != index) + lines.extend([f"This part holds {_part_range(part)}. The others: {others}.", ""]) + lines.extend(_console_install_block(console, (installs or {}).get(console, []))) + lines.extend(["## Files", ""]) + lines.extend(_cards_markdown(part)) + lines.append("") + lines.extend(_next_block([f"[{manufacturer}]({slug}.md), every section"] + + _NEXT_FROM_SYSTEMS)) + lines.append(f'
Generated on {_timestamp()}.
') + held = {sha1 for _variant, _card, sha1 in part} + pages[stem] = ("\n".join(lines) + "\n", {console: [ + f for f in consoles[console] if str(f.get("sha1", "")) in held]}) + summary.extend(_next_block(_NEXT_FROM_SYSTEMS)) + summary.append(f'
Generated on {_timestamp()}.
') + kept = {console: files for console, files in consoles.items() if console not in moved} + return {slug: ("\n".join(summary) + "\n", kept), **pages} + + +def _part_range(part: list[Card]) -> str: + """The first and last file names of a part, as the cards show them.""" + names = [re.search(r"`([^`]*)`", card[2]).group(1) for _variant, card, _sha1 in part] + if len(names) == 1: + return f"`{names[0]}`" + return f"{_count(len(names), 'file')}, `{names[0]}` to `{names[-1]}`" + + def generate_system_page( manufacturer: str, consoles: dict[str, list], @@ -3245,52 +3673,10 @@ def generate_system_page( """One manufacturer's page, a section per console. ``installs`` gives, per console, the (platform, system id) pairs whose - install command the section offers. + install command the section offers. A page over the budget is the + summary system_pages writes, its consoles on pages of their own. """ - slug = manufacturer.lower().replace(" ", "-") - lines = [ - f"# {manufacturer}", - "", - f"The files below by console, with every hash, are also published as " - f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).", - "", - ] - anchored: set[str] = set() - - for console_name in sorted(consoles.keys()): - files = consoles[console_name] - icon_md = system_icon_markdown(manufacturer, console_name) - lines.append(f"## {icon_md}{console_name}") - lines.append("") - lines.extend(_console_install_block(console_name, (installs or {}).get(console_name, []))) - main_files = [f for f in files if "/.variants/" not in f["path"]] - variant_files = [f for f in files if "/.variants/" in f["path"]] - - for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers, anchored)) - - if variant_files: - # A variant gets the same card as a primary: the revision a - # platform pins is the one a reader comes to find, and showed as - # a name and a truncated MD5, nothing said who pins it. - lines.append(f'??? note "Variants ({len(variant_files)})"') - lines.append("") - for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): - lines.extend( - f" {line}" if line else "" - for line in _system_file_card(v, consumers, anchored) - ) - lines.append("") - - lines.append("") - - lines.extend(_next_block([ - "[Get your files](../which-pack.md), for the setup you play on", - "[When a game does not start](../wiki/troubleshooting.md)", - "[Every emulator](../emulators/index.md), and the files each one reads", - ])) - lines.append(f'
Generated on {_timestamp()}.
') - return "\n".join(lines) + "\n" + return system_pages(manufacturer, consoles, consumers, installs)[_slug(manufacturer)][0] # Emulator pages @@ -3731,7 +4117,7 @@ def _render_emulator_file( # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. - page = parts[1].lower().replace(" ", "-") + page = card_page(held, sha1) details.append( f"Collection copy: [{_code(held)}]" f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" @@ -3805,7 +4191,7 @@ def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: archive = _code(path) zsha1 = consumers.sha1_at(path) if zsha1 and len(parts) >= 3 and parts[0] == "bios": - page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) + page = urllib.parse.quote(card_page(path, zsha1)) archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" return f"Collection copy: member {_code(member)} of {archive}" if parts[0] == "data" and len(parts) >= 2: @@ -3837,8 +4223,7 @@ def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: parts = folder.split("/") where = _code(folder + "/") if len(parts) >= 3 and parts[0] == "bios": - page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) - where = f"[{where}](../systems/{page}.md#{_slugify_anchor(parts[2])})" + where = f"[{where}]({console_href(_slug(parts[1]), parts[2], '../')})" tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(folder)}" return [f"Collection copies: {_count(len(members), 'file')} under {where} " f"([browse]({tree}))"] @@ -3907,14 +4292,14 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - return [ + return search_excluded([ '??? tip "Install this emulator\'s files"', "", " The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", *install_commands(f"--emulator {name}", " "), - ] + ]) def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]: @@ -3975,7 +4360,7 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" for platform, system in declared ] - return lines + [" ```", ""] + return search_excluded(lines + [" ```", ""]) def _render_platform_details(profile: dict) -> list[str]: @@ -4186,6 +4571,41 @@ _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} +# An emulator page lists at most this many file cards: MAME's 1,564 made a +# 3 MB page. Past it, the cards go to pages of their own and the emulator's +# page keeps a table of every entry, each row holding the card's old +# anchor. A count, not a size: system cards link to these cards and are +# written before them. +EMULATOR_CARDS_PER_PAGE = 400 + + +def emulator_parts(name: str, files: list) -> list[tuple[str, range]]: + """The pages an emulator's cards go to, with the entries each holds, + or none when they fit on the emulator's page. Balanced, in profile order.""" + if len(files) <= EMULATOR_CARDS_PER_PAGE: + return [] + count = -(-len(files) // EMULATOR_CARDS_PER_PAGE) + size = -(-len(files) // count) + return [(f"{name}-files-{index + 1}", range(start, min(start + size, len(files)))) + for index, start in enumerate(range(0, len(files), size))] + + +def plan_emulator_pages(profiles: dict) -> None: + """Record which page holds each emulator card, before any page links one.""" + stems = set() + for name, profile in sorted(profiles.items()): + if profile.get("type") in ("alias", "test"): + continue + files = profile.get("files") or [] + anchors = emulator_file_anchors(files) + for stem, entries in emulator_parts(name, files): + stems.add(stem) + record_emulator_cards(name, stem, [anchors[i] for i in entries]) + clash = stems & set(profiles) + if clash: + raise RuntimeError(f"an emulator's card page takes a profile's address: {sorted(clash)}") + + def generate_emulator_page( name: str, profile: dict, @@ -4196,13 +4616,28 @@ def generate_emulator_page( data_dirs: list[dict] | None = None, ) -> str: """One emulator's page. ``data_dirs`` defaults to the profile's own.""" + return emulator_pages( + name, profile, db, consumers, data_names, data_registry, data_dirs)[name] + + +def emulator_pages( + name: str, + profile: dict, + db: dict, + consumers: FileConsumers | None = None, + data_names: set[str] | None = None, + data_registry: dict | None = None, + data_dirs: list[dict] | None = None, +) -> dict[str, str]: + """An emulator's page by stem under emulators/, and the pages its cards + went to when there are more than EMULATOR_CARDS_PER_PAGE.""" if profile.get("type") == "alias": parent = profile.get("alias_of", profile.get("bios_identical_to", "unknown")) - return ( + return {name: ( f"# {name}\n\n" f"This core uses the same firmware as **{parent}**.\n\n" f"See [{parent}]({parent}.md) for details.\n" - ) + )} emu_name = profile.get("emulator", name) emu_type = profile.get("type", "unknown") @@ -4236,12 +4671,13 @@ def generate_emulator_page( if data_dirs is None: data_dirs = profile.get("data_directories", []) + part_pages: dict[str, str] = {} lines = [ f"# {emu_name}", "", '
', "", - "| | |", + "| Detail | Value |", "|---|---|", f"| Type | {emu_type} |", ] @@ -4339,19 +4775,30 @@ def generate_emulator_page( else: _file_available = _availability_check(db, data_names) + lines.extend(["## Files", ""]) lines.extend(_emulator_file_summary(files, _file_available)) anchors = emulator_file_anchors(files) - for f, anchor, alias in zip(files, anchors, emulator_file_aliases(files, anchors)): - lines.extend( - _render_emulator_file( - f, profile, consumers, files, _file_available, anchor, name, - alias, - ) - ) + aliases = emulator_file_aliases(files, anchors) + cards = [ + _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name, alias) + for f, anchor, alias in zip(files, anchors, aliases) + ] + parts = emulator_parts(name, files) + if not parts: + for card in cards: + lines.extend(card) + else: + lines.extend(_card_index(emu_name, files, anchors, aliases, parts)) + for number, (stem, entries) in enumerate(parts, 1): + part_pages[stem] = _emulator_part_page( + name, emu_name, number, parts, [cards[i] for i in entries], + [files[i] for i in entries]) if data_dirs: - lines.extend(["## Data directories", ""]) + # The member tables are a file list with hashes: out of the search, + # dolphin, primehack and ishiiruka each put 210 KB of it there. + lines.extend(["## Data directories { data-search-exclude }", ""]) member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in @@ -4408,6 +4855,63 @@ def generate_emulator_page( "[When a game does not start](../wiki/troubleshooting.md)", ])) lines.extend([f"*Generated on {_timestamp()}*"]) + return {name: "\n".join(lines) + "\n", **part_pages} + + +def _card_index( + emu_name: str, files: list[dict], anchors: list[str], aliases: list[str], + parts: list[tuple[str, range]], +) -> list[str]: + """Every entry of a split emulator page, one row each, linked to its card. + + Each row holds the ids the card had on this page, so a link cited + before the split lands on the row, and site.js follows it to the card. + """ + page_of = {index: stem for stem, entries in parts for index in entries} + lines = [ + f"{emu_name} reads more files than one page holds: each entry below " + f"links to its card, on {_count(len(parts), 'page')} by profile order " + + ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1)) + + ".", + "", + '
', + "", + "| File | Required | Category |", + "|------|----------|----------|", + ] + for index, (f, anchor, alias) in enumerate(zip(files, anchors, aliases)): + ids = "".join(f'' for a in (anchor, alias) if a) + shown = str(f.get("path") or f.get("name", "")) + lines.append( + f"| {ids}[{_code(shown)}]({page_of[index]}.md#{anchor}) " + f"| {'yes' if f.get('required') else 'no'} | {f.get('category', 'bios')} |" + ) + lines.extend(["", "
", ""]) + return lines + + +def _emulator_part_page( + name: str, emu_name: str, number: int, parts: list[tuple[str, range]], + cards: list[list[str]], files: list[dict], +) -> str: + """One page of a split emulator's cards.""" + first = str(files[0].get("path") or files[0].get("name", "")) + last = str(files[-1].get("path") or files[-1].get("name", "")) + others = ", ".join( + f"[{n}]({stem}.md)" for n, (stem, _e) in enumerate(parts, 1) if n != number) + lines = [ + f"# {emu_name} files, part {number} of {len(parts)}", + "", + f"Part of [{emu_name}]({name}.md), whose page has the profile, the install " + f"command and every entry. This part holds {_count(len(files), 'entry', 'entries')}, " + f"{_code(first)} to {_code(last)}, in the profile's order. The other parts: {others}.", + "", + "## Files", + "", + *(line for card in cards for line in card), + ] + lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"])) + lines.extend([f"*Generated on {_timestamp()}*"]) return "\n".join(lines) + "\n" @@ -5762,19 +6266,35 @@ def nav_emulators(profiles: dict) -> dict[str, dict]: def _emulator_nav(profiles: dict) -> list: - """Emulators in alphabetical groups ("A to C"), by the name a user knows.""" - groups: dict[str, list[tuple[str, str]]] = {} + """Emulators in alphabetical groups ("A to C"), then by initial. + + Material writes the open section of the navigation into every page: a + group of 96 emulators put 65 KB of links before each emulator page's + content. One initial is at most 60. + """ + groups: dict[str, dict[str, list[tuple[str, str]]]] = {} for key, profile in nav_emulators(profiles).items(): display = str(profile.get("emulator", key)) initial = display[:1].upper() letters = next((g for g in EMULATOR_NAV_GROUPS if initial in g), "") label = f"{letters[0]} to {letters[-1]}" if letters else "0 to 9" - groups.setdefault(label, []).append((display, f"emulators/{key}.md")) + sub = initial if letters else label + groups.setdefault(label, {}).setdefault(sub, []).append((display, f"emulators/{key}.md")) + + def entries(items: list[tuple[str, str]]) -> list[dict]: + return [{d: path} for d, path in sorted(items, key=lambda e: e[0].casefold())] + order = ["0 to 9"] + [f"{g[0]} to {g[-1]}" for g in EMULATOR_NAV_GROUPS] - return ["emulators/index.md"] + [ - {label: [{d: path} for d, path in sorted(groups[label], key=lambda e: e[0].casefold())]} - for label in order if label in groups - ] + nav: list = ["emulators/index.md"] + for label in order: + if label not in groups: + continue + subs = groups[label] + if len(subs) == 1: + nav.append({label: entries(next(iter(subs.values())))}) + else: + nav.append({label: [{sub: entries(subs[sub])} for sub in sorted(subs)]}) + return nav def _system_nav(manufacturers: dict) -> list: @@ -5840,7 +6360,7 @@ def generate_mkdocs_nav( {"Dump provenance": "provenance.md"}, {"Updated platform files": "exports.md"}, {"Data and API": "data.md"}, - {"Find a file by its SHA1": "files/index.md"}, + {"Find a file": "files/index.md"}, {"Data model": "wiki/data-model.md"}, {"Architecture": "wiki/architecture.md"}, {"Build your own packs": "wiki/advanced-usage.md"}, @@ -5956,19 +6476,13 @@ def _generate(args: argparse.Namespace) -> None: suppl_names = _build_supplemental_index() print("Computing platform coverage...") - coverages = {} - for name in sorted(platform_names): - try: - cov = compute_coverage( - name, args.platforms_dir, db, data_registry, - emulators_dir=args.emulators_dir, - ) - coverages[name] = cov - print( - f" {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})" - ) - except FileNotFoundError as e: - print(f" {name}: skipped ({e})", file=sys.stderr) + coverages = platform_coverages( + sorted(platform_names), args.platforms_dir, db, data_registry, + emulators_dir=args.emulators_dir, db_path=args.db, + report=lambda _name, cov: print( + f" {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})" + ), + ) print("Loading emulator profiles...") profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False) @@ -5995,14 +6509,8 @@ def _generate(args: argparse.Namespace) -> None: # report the same files. gap_report = build_emulator_gap_report(profiles, coverages, db, suppl_names) - print("Generating static API and bulk data exports...") - exports = generate_data_exports( - docs, db, coverages, profiles, stats, gap_report - ) - write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT) - PAGE_DATA["data.md"] = dataset_structured_data(exports, db) - # Build system_id -> manufacturer page map (needed by all generators) + plan_emulator_pages(profiles) print("Building system cross-reference map...") manufacturers = _group_by_manufacturer(db) set_system_page_map( @@ -6010,25 +6518,6 @@ def _generate(args: argparse.Namespace) -> None: ) print(f" {len(_system_page_map)} system IDs mapped to pages") - export_reports = load_export_reports() - _EXPORTED.update(report["platform"] for report in export_reports) - - print("Generating platform pages...") - write_if_changed( - str(docs / "platforms" / "index.md"), - generate_platform_index(coverages, registry), - ) - for name, cov in coverages.items(): - # The emulators this platform runs, not every one of the system. - runs = resolve_platform_cores(cov["config"], profiles) - write_if_changed( - str(docs / "platforms" / f"{name}.md"), - generate_platform_page(name, cov, registry, { - emu: data for emu, data in emulator_files.items() if emu in runs - }), - ) - PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) - print("Generating system pages...") prime_system_icons({ @@ -6043,16 +6532,55 @@ def _generate(args: argparse.Namespace) -> None: console_installs = system_install_choices(coverages) (docs / "files").mkdir(parents=True, exist_ok=True) write_if_changed(str(docs / "files" / "index.md"), generate_files_page()) + reset_console_pages() for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") - page = generate_system_page(mfr, consoles, consumers, console_installs.get(slug)) - PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) + pages = system_pages(mfr, consoles, consumers, console_installs.get(slug)) + for stem, (page, held) in pages.items(): + PAGE_DATA[f"systems/{stem}.md"] = system_structured_data(stem, held) + if stem != slug: + PART_PAGES.add(f"systems/{stem}.md") + write_if_changed(str(docs / "systems" / f"{stem}.md"), page) write_single_item( docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", system_export_item(slug, mfr, consoles), str(db.get("generated_at") or _timestamp()), ) - write_if_changed(str(docs / "systems" / f"{slug}.md"), page) + clash = split_system_stems() & {_slug(m) for m in manufacturers} + if clash: + raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}") + + print("Generating static API and bulk data exports...") + exports = generate_data_exports( + docs, db, coverages, profiles, stats, gap_report + ) + write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT) + PAGE_DATA["data.md"] = dataset_structured_data(exports, db) + + export_reports = load_export_reports() + _EXPORTED.update(report["platform"] for report in export_reports) + + print("Generating platform pages...") + write_if_changed( + str(docs / "platforms" / "index.md"), + generate_platform_index(coverages, registry), + ) + for name, cov in coverages.items(): + # The emulators this platform runs, not every one of the system. + runs = resolve_platform_cores(cov["config"], profiles) + moved: dict[str, str] = {} + write_if_changed( + str(docs / "platforms" / f"{name}.md"), + generate_platform_page(name, cov, registry, { + emu: data for emu, data in emulator_files.items() if emu in runs + }, moved), + ) + for stem, page in moved.items(): + if stem in coverages: + raise RuntimeError(f"a system page takes the platform {stem}'s address") + PART_PAGES.add(f"platforms/{stem}.md") + write_if_changed(str(docs / "platforms" / f"{stem}.md"), page) + PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) print("Generating emulator pages...") _DATA_MEMBERS.update(data_directory_members()) @@ -6085,25 +6613,31 @@ def _generate(args: argparse.Namespace) -> None: indent=2, sort_keys=True, ensure_ascii=False, default=str, ) + "\n", ) - page = generate_emulator_page( + pages = emulator_pages( name, profile, db, consumers, suppl_names, data_registry, data_dirs=page_data_directories(name, profiles), ) PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile) - write_if_changed(str(docs / "emulators" / f"{name}.md"), page) + files = profile.get("files") or [] + for stem, entries in emulator_parts(name, files): + PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data( + {**profile, "files": [files[i] for i in entries]}) + PART_PAGES.add(f"emulators/{stem}.md") + for stem, page in pages.items(): + write_if_changed(str(docs / "emulators" / f"{stem}.md"), page) print("Generating cross-reference page...") write_if_changed( str(docs / "cross-reference.md"), - generate_cross_reference(coverages, profiles) + REFERENCE_NEXT, + tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT, ) print("Generating gap analysis page...") write_if_changed( str(docs / "gaps.md"), - generate_gap_analysis( + tables_out_of_search(generate_gap_analysis( profiles, coverages, db, suppl_names, registry, gap_report - ) + REFERENCE_NEXT, + )) + REFERENCE_NEXT, ) print("Generating updated platform files page...") @@ -6119,7 +6653,9 @@ def _generate(args: argparse.Namespace) -> None: provenance_report = build_report(db, load_provenance_snapshots()) write_if_changed( str(docs / "provenance.md"), - generate_provenance_page(db, provenance_report, load_derivable()) + REFERENCE_NEXT, + tables_out_of_search( + generate_provenance_page(db, provenance_report, load_derivable()) + ) + REFERENCE_NEXT, ) # Wiki pages: copy manually maintained sources + generate dynamic ones @@ -6142,6 +6678,11 @@ def _generate(args: argparse.Namespace) -> None: print("Generating contributing page...") write_if_changed(str(docs / "contributing.md"), generate_contributing()) + # Swept before the decoration, which rewrites every page it finds: a + # page no generator produced any more (a console moved back onto its + # manufacturer's page) was decorated, recorded, and kept for ever. + stale = _sweep_generated(docs, GENERATED_DIRS) + print("Adding page metadata and structured data...") decorate_markdown_pages(docs) @@ -6240,19 +6781,17 @@ markdown_extensions: - tables - toc: permalink: true - # File cards are level 4 headings: the search lands on them, and a - # page of a hundred cards keeps a readable table of contents. + # Level 3 is the wiki's subsections. On pages of file cards, also level + # 3, the hook keeps the table of contents to level 2. toc_depth: 3 - pymdownx.details - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.keys -- pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format +# No mermaid fence: Material drew it in the browser with a script from a +# CDN. Diagrams are SVG files written by scripts/sitediagrams.py. +- pymdownx.superfences - pymdownx.tabbed: alternate_style: true plugins: @@ -6268,26 +6807,11 @@ validation: config_path, config_text = _mkdocs_config(Path(docs), mkdocs_static + nav_yaml) write_if_changed(str(config_path), config_text) - total_pages = ( - 1 # home - + 1 - + len(coverages) # platform index + detail - + 1 - + len(manufacturers) # system index + detail - + 1 # cross-reference - + 1 - + sum( - 1 for profile in profiles.values() - if profile.get("type") not in ("alias", "test") - ) # emulator detail pages (aliases/tests remain metadata-only) - + 1 # gap analysis - + 1 # which-pack - + len(list(Path("wiki").glob("*.md"))) # wiki pages copied verbatim - + 1 # generated wiki/data-model - + 1 # contributing - + 1 # data and API + # Counted on disk: the split pages made a sum of page kinds wrong. + total_pages = sum( + 1 for path in docs.rglob("*.md") + if not any(part in LOCAL_NOTE_DIRS for part in path.relative_to(docs).parts[:-1]) ) - stale = _sweep_generated(docs, GENERATED_DIRS) print(f"\nGenerated {total_pages} pages in {args.docs_dir}/") if stale: print(f"Removed {stale} page(s) nothing produced this run") @@ -6319,6 +6843,13 @@ from siterender import ( # noqa: E402,F401 _system_page_map, _system_link, set_system_page_map, + card_page, + console_href, + record_console_pages, + record_emulator_cards, + emulator_card_page, + reset_console_pages, + split_system_stems, _platform_link, _emulator_link, _fmt_size, diff --git a/scripts/mkdocs_hooks.py b/scripts/mkdocs_hooks.py index 0028a3e1..11cef35d 100644 --- a/scripts/mkdocs_hooks.py +++ b/scripts/mkdocs_hooks.py @@ -10,8 +10,28 @@ from __future__ import annotations from typing import Any +# Pages made of file cards, level 3 headings by the hundred: their table of +# contents stops at level 2, the consoles of a system page, the Files of an +# emulator, the sections of a platform page (which listed every system id). +CARD_PAGES = ("systems/", "emulators/", "platforms/") + def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001 """Take lastmod from the page's own date, or leave it out.""" page.update_date = str(page.meta.get("date_modified") or "") return markdown + + +def _prune(items: list, depth: int) -> None: + for item in items: + if item.level >= depth: + item.children = [] + else: + _prune(item.children, depth) + + +def on_page_content(html: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001 + """Keep the table of contents of a page of cards to level 2.""" + if page.file.src_uri.startswith(CARD_PAGES): + _prune(page.toc.items, 2) + return html diff --git a/scripts/siterender.py b/scripts/siterender.py index f0b1cbc1..0683ad43 100644 --- a/scripts/siterender.py +++ b/scripts/siterender.py @@ -139,13 +139,75 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None: return _norm_page_map.get(_norm_system_id(sys_id)) +# A manufacturer page too large for one page is a summary, each console on +# pages of its own: (slug, console) -> its page stems under systems/, and +# (slug, console, sha1) -> the stem holding that card. A console of a page +# kept whole has no entry, its section is on the manufacturer's page. +_console_pages: dict[tuple[str, str], list[str]] = {} +_card_pages: dict[tuple[str, str, str], str] = {} + + +def reset_console_pages() -> None: + _console_pages.clear() + _card_pages.clear() + + +def record_console_pages( + slug: str, console: str, stems: list[str], sha1s: list[list[str]] +) -> None: + """Where a split manufacturer page put one console: its page stems, in + order, and the sha1 of each card on each of them.""" + _console_pages[(slug, console)] = list(stems) + for stem, part in zip(stems, sha1s): + for sha1 in part: + _card_pages.setdefault((slug, console, sha1), stem) + + +def split_system_stems() -> set[str]: + """Every page stem of a console moved to a page of its own.""" + return {stem for stems in _console_pages.values() for stem in stems} + + +def console_href(slug: str, console: str, prefix: str = "") -> str: + """Where a console's files are: its own page, or its section of the + manufacturer's page (also where a console cut into parts lists them).""" + stems = _console_pages.get((slug, console)) + if stems and len(stems) == 1: + return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md" + return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}" + + +def card_page(path: str, sha1: str) -> str: + """The systems/ page stem holding the card of the copy at ``path`` + (bios///...).""" + parts = path.split("/") + slug = parts[1].lower().replace(" ", "-") + console = parts[2] if len(parts) > 2 else "" + return _card_pages.get((slug, console, sha1), slug) + + +# Emulator cards moved off a page too long for one: (emulator, anchor) -> +# the page stem under emulators/ that holds the card. +_emulator_card_pages: dict[tuple[str, str], str] = {} + + +def record_emulator_cards(emulator: str, stem: str, anchors: list[str]) -> None: + for anchor in anchors: + _emulator_card_pages[(emulator, anchor)] = stem + + +def emulator_card_page(emulator: str, anchor: str) -> str: + """The emulators/ page stem holding a card: the emulator's own page, or + the page its cards went to.""" + return _emulator_card_pages.get((emulator, anchor), emulator) + + def _system_link(sys_id: str, prefix: str = "") -> str: """Generate a markdown link to a system page with anchor.""" page = _system_page_map.get(sys_id) or _page_by_norm(sys_id) if page: slug, console = page - anchor = _slugify_anchor(console) - return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})" + return f"[{sys_id}]({console_href(slug, console, prefix)})" return sys_id def _field_text(key: object, value: object) -> str: diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index c968d30b..80ed62e6 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -446,7 +446,7 @@ def main() -> int: if (api_dir / name).exists() ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( (api_dir / "systems").glob("*.json") - ) + ) + sorted((api_dir / "lookup").glob("*.json")) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/scripts/validate_site.py b/scripts/validate_site.py index d7f95b7a..b39bf9ee 100644 --- a/scripts/validate_site.py +++ b/scripts/validate_site.py @@ -34,7 +34,25 @@ class Page: jsonld: list[str] = field(default_factory=list) unrendered: list[str] = field(default_factory=list) remote_images: list[str] = field(default_factory=list) + remote_scripts: list[str] = field(default_factory=list) placeholders: list[str] = field(default_factory=list) + headings: list[int] = field(default_factory=list) + empty_headers: int = 0 + runtime_diagrams: int = 0 + size: int = 0 + + +# Budgets the site was brought under on 2026-10-22, with room to grow. A +# change that passes one is a regression of what a visitor downloads, not a +# matter of taste: raise the number in the same change that explains why. +# +# The largest page was 10.5 MB (systems/game-engines/) and took 21 s of +# main-thread work on a throttled phone. Split pages now top out at 0.9 MB. +MAX_PAGE_BYTES = 1_000_000 +# The search index is downloaded by every visitor who opens the search: +# 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without. +MAX_SEARCH_INDEX_BYTES = 8_500_000 +MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000 # Text inside these is shown as written, so markup there is not a leftover. @@ -50,11 +68,21 @@ class _PageParser(HTMLParser): self._jsonld_depth = 0 self._jsonld_parts: list[str] = [] self._literal_depth = 0 + self._header_parts: list[str] | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - data = dict(attrs) + data = {key: value or "" for key, value in attrs} if tag in _LITERAL_TAGS: self._literal_depth += 1 + if re.fullmatch(r"h[1-6]", tag): + self.page.headings.append(int(tag[1])) + if tag == "th": + self._header_parts = [] + if "mermaid" in data.get("class", "").split(): + # Material renders it in the browser with a script from a CDN. + self.page.runtime_diagrams += 1 + if tag == "script" and re.match(r"\s*(?:https?:)?//", data.get("src", "")): + self.page.remote_scripts.append(data["src"]) if tag == "html": self.page.lang = data.get("lang", "") elif tag == "title": @@ -97,6 +125,10 @@ class _PageParser(HTMLParser): def handle_endtag(self, tag: str) -> None: if tag in _LITERAL_TAGS and self._literal_depth: self._literal_depth -= 1 + if tag == "th" and self._header_parts is not None: + if not "".join(self._header_parts).strip(): + self.page.empty_headers += 1 + self._header_parts = None if tag == "title": self._title_depth -= 1 elif tag == "script" and self._jsonld_depth: @@ -105,6 +137,8 @@ class _PageParser(HTMLParser): self._jsonld_parts = [] def handle_data(self, data: str) -> None: + if self._header_parts is not None: + self._header_parts.append(data) if self._title_depth: self._title_parts.append(data) if self._jsonld_depth: @@ -119,11 +153,39 @@ class _PageParser(HTMLParser): def _parse_page(path: Path) -> Page: parser = _PageParser(path) - parser.feed(path.read_text(encoding="utf-8", errors="replace")) + text = path.read_text(encoding="utf-8", errors="replace") + parser.feed(text) parser.close() + parser.page.size = len(text.encode("utf-8")) return parser.page +def _heading_jumps(levels: list[int]) -> list[str]: + """Each heading more than one level below the one before it: a screen + reader user moving by level finds nothing at the level skipped.""" + return [f"h{before} to h{after}" for before, after in zip(levels, levels[1:]) + if after > before + 1] + + +def _check_search_index(site: Path) -> list[str]: + """The index every visitor who opens the search downloads, against its budget.""" + import gzip + + path = site / "search" / "search_index.json" + if not path.is_file(): + return [] + raw = path.read_bytes() + packed = len(gzip.compress(raw, 6)) + issues = [] + if len(raw) > MAX_SEARCH_INDEX_BYTES: + issues.append(f"search/search_index.json: {len(raw):,} bytes, " + f"over the {MAX_SEARCH_INDEX_BYTES:,} budget") + if packed > MAX_SEARCH_INDEX_GZIP_BYTES: + issues.append(f"search/search_index.json: {packed:,} bytes gzip, " + f"over the {MAX_SEARCH_INDEX_GZIP_BYTES:,} budget") + return issues + + def _base_path(config_path: Path) -> str: """Read site_url without constructing executable YAML extension tags.""" config = config_path.read_text(encoding="utf-8") @@ -220,6 +282,17 @@ def _check_pages( report(path, f"images missing alt: {page.missing_alt[:5]}") if page.remote_images: report(path, f"images loaded from another host: {page.remote_images[:3]}") + if page.remote_scripts: + report(path, f"scripts loaded from another host: {page.remote_scripts[:3]}") + if page.runtime_diagrams: + report(path, f"{page.runtime_diagrams} diagram(s) drawn by a script at runtime") + if page.size > MAX_PAGE_BYTES: + report(path, f"{page.size:,} bytes of HTML, over the {MAX_PAGE_BYTES:,} budget") + jumps = _heading_jumps(page.headings) + if jumps: + report(path, f"heading levels skipped: {sorted(set(jumps))}") + if page.empty_headers: + report(path, f"{page.empty_headers} table header cell(s) without text") if page.placeholders: # scripts/sitediagrams.py knows no diagram of this name. report(path, f"image placeholders left unresolved: {page.placeholders[:3]}") @@ -294,6 +367,7 @@ def validate_site(site: Path, config_path: Path) -> list[str]: issues.append(f"duplicate {label} {value!r}: {rendered}") issues.extend(_check_links(pages, site, base_path)) + issues.extend(_check_search_index(site)) return issues diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index f189f963..58b44745 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -779,9 +779,11 @@ class FaqRegressions(unittest.TestCase): @staticmethod def _faq() -> str: - # The questions about how files are identified and checked moved to - # the technical FAQ when the site was laid out by level. - return (ROOT / "wiki" / "technical-faq.md").read_text(encoding="utf-8") + # The questions about hashes and checks moved to the technical FAQ. + return "\n\n".join( + (ROOT / "wiki" / name).read_text(encoding="utf-8") + for name in ("faq.md", "technical-faq.md") + ) @staticmethod def _profile(key: str) -> dict: diff --git a/tests/test_coverage_cache.py b/tests/test_coverage_cache.py new file mode 100644 index 00000000..b9b38398 --- /dev/null +++ b/tests/test_coverage_cache.py @@ -0,0 +1,100 @@ +"""The site and the README share one coverage computation. + +generate_site.py and generate_readme.py each verified every platform, a +minute of the site build spent twice on the same answer. The first to run +stores the coverage, keyed by its inputs, and the second reads it. A cache +that outlived a change to its inputs would publish stale counts, so these +tests change an input and expect a fresh computation. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_readme # noqa: E402 + + +class CoverageCache(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) + self.platforms = self.work / "platforms" + self.platforms.mkdir() + (self.platforms / "alpha.yml").write_text("platform: Alpha\n") + self.emulators = self.work / "emulators" + self.emulators.mkdir() + self.db = self.work / "database.json" + self.db.write_text("{}") + self.calls: list[str] = [] + patches = [ + mock.patch.object(generate_readme, "COVERAGE_CACHE_DIR", self.work / "cache"), + mock.patch.object(generate_readme, "compute_coverage", self._compute), + ] + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + def tearDown(self) -> None: + import shutil + + shutil.rmtree(self.work, ignore_errors=True) + + def _compute(self, name, platforms_dir, db, data_registry=None, emulators_dir=""): + self.calls.append(name) + if name == "ghost": + raise FileNotFoundError(name) + return {"platform": name.title(), "missing_names": {"a.bin"}} + + def _run(self, names=("beta", "alpha")) -> dict: + return generate_readme.platform_coverages( + list(names), str(self.platforms), {}, {}, + emulators_dir=str(self.emulators), db_path=str(self.db), + ) + + def test_second_run_reads_the_first(self): + first = self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + second = self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + self.assertEqual(first, second) + # The caller's order, not the computation's. + self.assertEqual(list(second), ["beta", "alpha"]) + self.assertEqual(second["alpha"]["missing_names"], {"a.bin"}) + + def test_a_changed_input_computes_again(self): + self._run() + for change in ( + lambda: (self.platforms / "alpha.yml").write_text("platform: Alpha 2\n"), + lambda: self.db.write_text('{"files": {}}'), + lambda: (self.emulators / "core.yml").write_text("emulator: core\n"), + ): + self.calls.clear() + change() + self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + + def test_another_platform_list_computes_again(self): + self._run() + self.calls.clear() + self._run(("alpha",)) + self.assertEqual(self.calls, ["alpha"]) + + def test_a_platform_without_a_file_is_left_out(self): + self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"]) + self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"]) + + def test_one_cache_file_is_kept(self): + self._run() + (self.platforms / "alpha.yml").write_text("platform: Alpha 3\n") + self._run() + self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_entry_categories.py b/tests/test_entry_categories.py index d6cbb324..3d2791a9 100644 --- a/tests/test_entry_categories.py +++ b/tests/test_entry_categories.py @@ -13,7 +13,14 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) import verify # noqa: E402 -from exporter.candidates import BIOS, GAME_DATA, UNREAD, EntryCategories # noqa: E402 +from exporter.candidates import ( # noqa: E402 + BIOS, + GAME_DATA, + UNREAD, + EntryCategories, + ProofIndex, + ReadEvidence, +) PROFILES = { "bluemsx": { @@ -144,5 +151,49 @@ class StrictCountsFirmwareOnly(unittest.TestCase): self.assertEqual(evidence["diabdat.mpq"]["read_by"][0]["emulator"], "devilutionx") +class IndexedProof(unittest.TestCase): + """ProofIndex answers what ReadEvidence.proof answers, reader by reader.""" + + def test_every_kind_matches_the_direct_proof(self): + md5 = "490f666e1afb15b7362b406ed1cea246" + readers = [] + for profile in ( + {"files": [{"name": "a.bin", "path": "sys/dir/a.bin"}]}, + {"files": [{"name": "other.rom", "aliases": ["B.ROM"]}]}, + {"files": [{"name": "c.bin", "md5": md5}]}, + {"files": [{"name": "tree/", "type": "directory", "path": "data/tree/"}]}, + {"files": [], "data_directories": [{"ref": "x", "destination": "assets/sys"}]}, + {"files": [{"name": "d.bin", "sha1": "ab" * 20, "crc32": "12345678"}]}, + ): + evidence = ReadEvidence() + evidence.add_profile(profile, {}) + readers.append(evidence) + launcher = ReadEvidence() + launcher.add_launchers([{"names": ["scph101.bin"]}]) + readers.append(launcher) + index = ProofIndex(readers) + cases = [ + ({"name": "a.bin"}, "bios/sys/dir/a.bin"), + ({"name": "a.bin"}, "elsewhere/a.bin"), + ({"name": "x", "aliases": ["b.rom"]}, "x"), + ({"name": "y", "md5": md5[:29]}, "y"), + ({"name": "y", "md5": md5.upper()}, "y"), + ({"name": "z", "md5": md5[:28]}, "z"), + ({"name": "z"}, "root/data/tree/deep/z"), + ({"name": "z"}, "data/tree"), + ({"name": "z"}, "a/assets/sys/z.png"), + ({"name": "scph101.bin"}, "bios/scph101.bin"), + ({"name": "q", "crc32": "12345678, 9"}, "q"), + ({"name": "q"}, ""), + ] + for entry, destination in cases: + expected = { + position: proof for position, evidence in enumerate(readers) + if (proof := evidence.proof(entry, destination)) is not None + } + with self.subTest(entry=entry, destination=destination): + self.assertEqual(index.proofs(entry, destination), expected) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 07a31239..be861d5c 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -651,6 +651,62 @@ class APerCoreDocumentValidates(unittest.TestCase): jsonschema.validate(document["items"][0]["profile"], schema) +class AHashFindsEveryPageThatNamesIt(unittest.TestCase): + """Hashes left the site search, 4 MB of the index every visitor + downloaded. files/?hash= reads one shard instead, and it must answer at + least what the search answered: the card, the platforms, the profiles.""" + + def test_records_cover_every_layer_and_shard_by_prefix(self): + import json as _json + import tempfile + + import generate_site # noqa: PLC0415 + + sha1 = "ab" * 20 + md5 = "490f666e1afb15b7362b406ed1cea246" + db = {"files": {sha1: {"name": "scph5501.bin", "md5": md5, "crc32": "8D8CB7E4", + "path": "bios/Sony/PlayStation/scph5501.bin"}}} + coverages = {"batocera": {"config": {"systems": {"sony-playstation": {"files": [ + {"name": "scph5501.bin", "md5": f"{md5}, 0123456789abcdef0123456789abcdef"}, + {"name": "typo.bin", "md5": "not-a-hash"}, + ]}}}}} + profiles = { + "duckstation": {"files": [{"name": "scph5501.bin", "md5": md5}, + {"name": "set.zip", "contents": [ + {"name": "m.rom", "crc32": "11223344"}]}]}, + "an-alias": {"type": "alias", "files": [{"name": "x", "md5": md5}]}, + } + locations = generate_site.file_location_items(db) + [ + {"sha1": "cd" * 20, "url": "emulators/dolphin/#sha1-" + "cd" * 6}] + members = {"dolphin-sys": {"Sys/GC/font.bin": {"sha1": "cd" * 20}}} + records = generate_site.file_lookup_records(db, coverages, profiles, locations, members) + found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records} + self.assertEqual(found[("collection", "", "md5", md5)]["url"], + "systems/sony/#sha1-" + "ab" * 6) + self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found) + self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin") + self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"], + "platforms/batocera/#sony-playstation") + self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found) + self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"], + "emulators/duckstation/#file-scph5501-bin-490f666e") + self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom") + self.assertNotIn(("emulator", "an-alias", "md5", md5), found) + + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + shards = generate_site.write_lookup_shards(Path(tmp), records, "2026-01-01T00:00:00Z") + folder = Path(tmp) / "lookup" + names = sorted(path.name for path in folder.glob("*.json")) + self.assertEqual(len(names), shards) + for path in folder.glob("*.json"): + document = _json.loads(path.read_text()) + self.assertEqual(document["kind"], "file-lookup") + self.assertEqual(document["count"], len(document["items"])) + for item in document["items"]: + self.assertTrue(item["value"].startswith(path.stem)) + self.assertNotIn("no.json", names) + + class AFileHasAnAddressByContent(unittest.TestCase): """mt32_control.rom moved from the Commodore page to the Roland page and every link to its card broke: the anchor was content-based, the page @@ -669,7 +725,8 @@ class AFileHasAnAddressByContent(unittest.TestCase): ) self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page()) script = (Path(__file__).resolve().parents[1] / "docs_assets" / "site.js").read_text() - self.assertIn("file-locations.json", script) + self.assertIn("api/v1/lookup/", script) + self.assertIn('params.has("sha1")', script) from cross_reference import FileConsumers # noqa: PLC0415 card = "\n".join(generate_site._system_file_card( diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 78d9aeb5..a5b4d6f0 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -323,7 +323,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase): entry = {"name": "scph5501.bin", "required": True} lines = generate_site._render_emulator_file( entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin") - self.assertTrue(lines[2].startswith("#### `scph5501.bin` ")) + self.assertTrue(lines[2].startswith("### `scph5501.bin` ")) self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }")) def test_a_system_card_is_a_heading_with_its_anchor(self): @@ -332,16 +332,17 @@ class TheSearchLandsOnTheCard(unittest.TestCase): entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3} lines = generate_site._system_file_card(entry, FileConsumers({"files": {}})) - self.assertTrue(lines[2].startswith("#### `a.bin` ")) + self.assertTrue(lines[2].startswith("### `a.bin` ")) self.assertTrue(lines[2].endswith("{ #sha1-aaaaaaaaaaaa .rb-file-title }")) class TheSearchSkipsLinkLists(unittest.TestCase): """The search index was 24.7 MB, read and indexed before the first result. Lists of platforms, copies and source citations on every card - made 6.6 MB of it, names already indexed on their own pages.""" + made 6.6 MB of it, names already indexed on their own pages, and the + hashes 4 MB more: files/?hash= finds those.""" - def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self): + def test_a_system_card_keeps_its_name_and_skips_its_body(self): import generate_site from cross_reference import FileConsumers @@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase): consumers.emulators = {sha1: {"duckstation"}} entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3} text = "\n".join(generate_site._system_file_card(entry, consumers)) - searched, skipped = text.split('
') - self.assertIn("- MD5:", searched) - self.assertIn("- Emulators:", searched) - self.assertIn("- Platforms:", skipped) - self.assertIn("- Also installed by:", skipped) - self.assertEqual(text.count("
"), 2) + searched, skipped = text.split("
") + self.assertIn("`a.bin`", searched) + self.assertNotIn("- MD5:", searched) + for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"): + self.assertIn(line, skipped) + + def test_a_game_data_card_skips_its_name_too(self): + import generate_site + from cross_reference import FileConsumers + + entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3} + lines = generate_site._system_file_card( + entry, FileConsumers({"files": {}}), searchable=False) + self.assertTrue(lines[2].endswith("data-search-exclude }")) def test_runs_alternate_in_card_order(self): import generate_site - lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", - "- Destination: `d`", "- Contents (1 members):", " - `m`"] + lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", + "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- Note"] out = [line for line in generate_site._search_skipped_apart(lines) if line] self.assertEqual(out, [ - "- SHA1: x", + "- Path: `p`", '
', - "- Source: `a.c:1`", "- Platforms: p", "
", - "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "
", + "- Destination: `d`", + '
', + "- Contents (1 members):", " - `m`", "
", + "- Note", ]) + def test_long_tables_leave_the_search(self): + import generate_site + + short = "| a |\n|---|\n| 1 |" + long = " | a |\n |---|\n" + "\n".join(f" | {'x' * 50} |" for _ in range(80)) + text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n") + self.assertEqual(text.count("data-search-exclude"), 1) + self.assertIn("
\n\n | a |", text) + self.assertIn(short, text) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py index 7ebe6ca5..4c897af3 100644 --- a/tests/test_site_validation.py +++ b/tests/test_site_validation.py @@ -166,6 +166,67 @@ class RenderedSiteValidation(unittest.TestCase): +class TheSiteStaysWithinItsBudgets(unittest.TestCase): + """The 10.5 MB page and the 21 MB search index passed every check: + validate_site now holds the gains, each limit with its reason.""" + + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT) + self.site = Path(self.temp.name) / "site" + self.site.mkdir() + self.config = Path(self.temp.name) / "mkdocs.yml" + self.config.write_text("site_url: https://example.test/retrobios/\n", encoding="utf-8") + + def tearDown(self) -> None: + self.temp.cleanup() + + def _issues(self, body: str) -> list[str]: + (self.site / "index.html").write_text(_page("Home", "Home page.", body), encoding="utf-8") + return validate_site(self.site, self.config) + + def test_a_clean_page_passes(self): + (self.site / "javascripts").mkdir() + (self.site / "javascripts" / "site.js").write_text("", encoding="utf-8") + self.assertEqual(self._issues( + "

A

B

C

" + "
DetailValue
" + ''), []) + + def test_a_page_over_its_size(self): + import validate_site as module + + issues = self._issues("

" + "x" * module.MAX_PAGE_BYTES + "

") + self.assertTrue(any("over the" in issue and "HTML" in issue for issue in issues), issues) + + def test_a_skipped_heading_level(self): + issues = self._issues("

A

B

") + self.assertTrue(any("h2 to h4" in issue for issue in issues), issues) + + def test_an_empty_table_header(self): + issues = self._issues("
") + self.assertTrue(any("2 table header cell(s) without text" in issue for issue in issues)) + + def test_a_script_or_a_diagram_from_another_host(self): + issues = self._issues( + '' + '
graph LR
') + self.assertTrue(any("scripts loaded from another host" in issue for issue in issues)) + self.assertTrue(any("drawn by a script at runtime" in issue for issue in issues)) + + def test_a_search_index_over_its_budget(self): + import random + + import validate_site as module + + (self.site / "search").mkdir() + rng = random.Random(1) + noise = "".join(rng.choice("0123456789abcdef") for _ in range(module.MAX_SEARCH_INDEX_BYTES + 1)) + (self.site / "search" / "search_index.json").write_text(noise, encoding="utf-8") + issues = self._issues("") + self.assertTrue(any("search_index.json" in issue and "gzip" not in issue for issue in issues)) + self.assertTrue(any("search_index.json" in issue and "gzip" in issue for issue in issues)) + + class UnrenderedMarkdown(unittest.TestCase): """The clownmdemu analysis block reached readers as "- **bios_hle:** True": a list indented one level too deep, read as code.""" diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py new file mode 100644 index 00000000..a524d7ed --- /dev/null +++ b/tests/test_site_weight.py @@ -0,0 +1,168 @@ +"""Pages a phone can open: split pages, light navigation, short tables of contents. + +systems/game-engines/ was one 10.5 MB page, emulators/mame/ 3 MB, and each +of 600 emulator pages carried 65 KB of navigation. A large page is now a +summary with pages of its own, and every address cited before the split +still lands: a console's heading stays on the summary, a card's sha1 anchor +is looked up by content, an emulator card's anchor sits on its row. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import mkdocs_hooks # noqa: E402 +import siterender # noqa: E402 +from cross_reference import FileConsumers # noqa: E402 + + +def _file(name: str, index: int, console: str, maker: str = "Maker") -> dict: + sha1 = f"{index:012x}" + "0" * 28 + return {"name": name, "sha1": sha1, "md5": f"{index:032x}", "size": 10, + "path": f"bios/{maker}/{console}/{name}"} + + +class ASystemPageOverItsBudgetIsSplit(unittest.TestCase): + def setUp(self) -> None: + siterender.reset_console_pages() + self.addCleanup(siterender.reset_console_pages) + self.consoles = { + "Big": [_file(f"big{i:03d}.bin", i, "Big") for i in range(60)], + "Small": [_file("small.bin", 900, "Small")], + } + + def _pages(self, budget: int) -> dict: + return generate_site.system_pages( + "Maker", self.consoles, FileConsumers({"files": {}}), budget=budget) + + def test_a_page_within_its_budget_stays_whole(self): + pages = self._pages(10_000_000) + self.assertEqual(list(pages), ["maker"]) + self.assertIn("## Big", pages["maker"][0]) + + def test_the_largest_section_moves_and_is_cut_in_parts(self): + pages = self._pages(20_000) + summary = pages["maker"][0] + # Both headings stay on the summary: systems/maker/#big still lands. + self.assertIn("## Big", summary) + self.assertIn("## Small", summary) + # The small section keeps its cards in place. + self.assertIn("{ #sha1-" + f"{900:012x}", summary) + parts = [stem for stem in pages if stem.startswith("maker-big")] + self.assertGreater(len(parts), 1) + self.assertEqual(parts[0], "maker-big") + for stem in parts: + self.assertIn(f"]({stem}.md)", summary) + self.assertLessEqual(len(pages[stem][0]), 20_000 + 5_000) + self.assertIn("## Files", pages[stem][0]) + # Each card is on exactly one part, and the links follow it. + held = [f["sha1"] for stem in parts for f in pages[stem][1]["Big"]] + self.assertEqual(sorted(held), sorted(f["sha1"] for f in self.consoles["Big"])) + last = self.consoles["Big"][-1] + self.assertEqual(siterender.card_page(last["path"], last["sha1"]), parts[-1]) + small = self.consoles["Small"][0] + self.assertEqual(siterender.card_page(small["path"], small["sha1"]), "maker") + self.assertEqual(siterender.console_href("maker", "Big"), "systems/maker.md#big") + + def test_a_console_on_one_page_is_linked_directly(self): + self.consoles["Big"] = self.consoles["Big"][:12] + whole = self._pages(10**9)["maker"][0] + pages = self._pages(len(whole) - 1) + self.assertIn("maker-big", pages) + self.assertNotIn("maker-big-2", pages) + self.assertEqual(siterender.console_href("maker", "Big", "../"), + "../systems/maker-big.md") + + +class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase): + def test_parts_are_balanced_and_rows_keep_the_anchors(self): + files = [{"name": f"set{i:04d}.zip", "required": i % 2 == 0, "category": "bios_zip"} + for i in range(generate_site.EMULATOR_CARDS_PER_PAGE * 2 + 1)] + parts = generate_site.emulator_parts("big", files) + self.assertEqual([stem for stem, _ in parts], ["big-files-1", "big-files-2", "big-files-3"]) + sizes = [len(entries) for _, entries in parts] + self.assertLessEqual(max(sizes) - min(sizes), 1) + self.assertEqual(generate_site.emulator_parts("small", files[:3]), []) + + pages = generate_site.emulator_pages( + "big", {"emulator": "Big", "type": "standalone", "files": files}, {"files": {}}) + self.assertEqual(sorted(pages), ["big", "big-files-1", "big-files-2", "big-files-3"]) + main = pages["big"] + anchors = generate_site.emulator_file_anchors(files) + for anchor in anchors: + self.assertIn(f'', main) + self.assertIn('class="rb-card-index"', main) + self.assertNotIn(".rb-file-title", main) + self.assertIn(f"{{ #{anchors[0]} .rb-file-title }}", pages["big-files-1"]) + + def test_cards_are_linked_on_the_page_that_holds_them(self): + files = [{"name": f"set{i:04d}.zip"} for i in range(generate_site.EMULATOR_CARDS_PER_PAGE + 1)] + generate_site.plan_emulator_pages({"big": {"files": files}}) + anchors = generate_site.emulator_file_anchors(files) + self.assertEqual(siterender.emulator_card_page("big", anchors[-1]), "big-files-2") + self.assertEqual(siterender.emulator_card_page("other", "file-x"), "other") + + +class APlatformSystemOverItsBudgetMoves(unittest.TestCase): + def test_the_heading_stays_and_the_list_moves(self): + files = [{"name": f"f{i:04d}.zip", "status": "ok"} for i in range(40)] + config = {("arcade", f["name"]): [{"name": f["name"], "md5": "a" * 32}] for f in files} + moved: dict[str, str] = {} + budget = generate_site.PLATFORM_SECTION_BUDGET + generate_site.PLATFORM_SECTION_BUDGET = 500 + try: + lines = generate_site._render_platform_systems( + {"arcade": files}, config, {}, ("deck", "Deck"), moved) + finally: + generate_site.PLATFORM_SECTION_BUDGET = budget + text = "\n".join(lines) + self.assertIn("{ #arcade }", text) + self.assertIn("](deck-arcade.md)", text) + self.assertNotIn("f0001.zip", text) + self.assertIn("f0001.zip", moved["deck-arcade"]) + self.assertIn("[Deck page](deck.md#arcade)", moved["deck-arcade"]) + + +class TheNavigationOpensOneInitial(unittest.TestCase): + def test_a_group_of_several_initials_is_nested(self): + profiles = { + key: {"emulator": name, "systems": ["s"]} + for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed")) + } + nav = generate_site._emulator_nav(profiles) + groups = {next(iter(g)): next(iter(g.values())) for g in nav[1:]} + self.assertEqual(groups["A to C"], [ + {"A": [{"Alpha": "emulators/a1.md"}]}, + {"B": [{"Beta": "emulators/b1.md"}, {"bravo": "emulators/b2.md"}]}, + ]) + self.assertEqual(groups["V to Z"], [{"Zed": "emulators/z.md"}]) + + +class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase): + @staticmethod + def _item(level: int, children: list | None = None) -> SimpleNamespace: + return SimpleNamespace(level=level, children=children or []) + + def test_cards_leave_the_toc_and_wiki_pages_keep_level_three(self): + def page(uri: str) -> SimpleNamespace: + h2 = self._item(2, [self._item(3)]) + return SimpleNamespace(file=SimpleNamespace(src_uri=uri), + toc=SimpleNamespace(items=[self._item(1, [h2])])) + + for uri, kept in (("systems/sony.md", 0), ("platforms/retroarch.md", 0), + ("emulators/mame.md", 0), ("wiki/faq.md", 1)): + current = page(uri) + mkdocs_hooks.on_page_content("", current, None, None) + with self.subTest(uri=uri): + self.assertEqual(len(current.toc.items[0].children[0].children), kept) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/architecture.md b/wiki/architecture.md index c88d6554..3ab42b66 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -71,14 +71,32 @@ dist/ generated packs (gitignored) Documentation changes belong in `wiki/` or in the generator. The site search splits each page at its headings and sends a result to the -heading above the match. Every file card is therefore a level 4 heading with -the card's anchor, and every platform system a heading of its own: a hash -pasted in the search opens that card or that system, not the top of the page. -Level 4 stays out of the table of contents (`toc_depth: 3`). The lines that -list other pages (platforms, packs, source citations, collection paths) sit -in a `data-search-exclude` block, since the browser downloads and indexes the -whole search index before it shows a first result. File names, hashes, the -paths an emulator reads and the notes stay searchable. +heading above the match. Every file card is therefore a level 3 heading with +the card's anchor, under its console or a Files heading, and every platform +system a heading of its own: a file name typed in the search opens that card, +not the top of the page. On pages of cards (systems, emulators, platforms) the +table of contents stops at level 2 (`scripts/mkdocs_hooks.py`). + +The browser downloads and indexes the whole search index before it shows a +first result, so the index holds names and prose only. Hashes, lists of other +pages, install commands, member tables and long reference tables sit in +`data-search-exclude` blocks. Those blocks are `section` elements: the search +plugin compares elements by tag, and a `div` stopped excluding at the first +`div` closed inside it. A hash goes to [Find a file](../files/index.md), which +reads one shard of `api/v1/lookup/` named by the first two characters of the +value. + +A page has a budget. A manufacturer page over 500 KB of Markdown becomes a +summary: its largest consoles move to pages of their own, cut into parts when +one console alone is over the budget, and every console keeps its heading on +the summary. An emulator with more than 400 entries (MAME) lists them in a +table on its page, each row holding the old anchor of the card, and the cards +move to pages of 400. A platform system whose list is over 200 KB moves to a +page of its own. A card anchor (`#sha1-...`) that a page no longer holds is +looked up by content, so a link cited before a split still opens the card. +`validate_site.py` fails a page over 1 MB of HTML, a search index over 8.5 MB +(1.7 MB compressed), a skipped heading level, a table header without text, a +script from another host and a diagram drawn at runtime. ## Data flow diff --git a/wiki/index.md b/wiki/index.md index 755d1ee3..dfe00c8a 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -20,7 +20,7 @@ toolchain works. To set up BIOS files on a device, start with ## Data - **[Data and API](../data.md)** - JSON, CSV and SQLite exports with their checksums -- **[Find a file by its SHA1](../files/index.md)** - the permanent address of a file by its contents +- **[Find a file](../files/index.md)** - any SHA1, MD5, CRC32 or SHA256: the file, the platforms that declare it, the emulators that check it - **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats ## The toolchain diff --git a/wiki/release-process.md b/wiki/release-process.md index 68c81fb6..cfe47134 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -54,7 +54,8 @@ is missing. 6. Run `generate_readme.py` (rebuilds README.md and CONTRIBUTING.md) 7. `mkdocs build --strict` to produce the static site 8. Run `validate_site.py` on the rendered HTML (metadata, headings, image - alternatives, duplicate ids, local links and fragments) + alternatives, duplicate ids, local links and fragments, page and search + index budgets) 9. Require the committed README and CONTRIBUTING to match what the generator just produced. `write_if_changed()` compares content with the timestamp line stripped, so a run that only moves the clock leaves the files untouched and diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 5cf5e803..f3ee044c 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture diff --git a/wiki/tools.md b/wiki/tools.md index b31e73f6..719c025e 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -692,7 +692,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) | | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database | | `generate_site.py` | Generate all MkDocs site pages (this documentation) | -| `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments | +| `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets | | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held | | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit | | `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |