diff --git a/docs_assets/logos/amberelec.png b/docs_assets/logos/amberelec.png deleted file mode 100644 index ddc713f4..00000000 Binary files a/docs_assets/logos/amberelec.png and /dev/null differ diff --git a/docs_assets/logos/amberelec.webp b/docs_assets/logos/amberelec.webp new file mode 100644 index 00000000..f5176139 Binary files /dev/null and b/docs_assets/logos/amberelec.webp differ diff --git a/docs_assets/logos/bizhawk.png b/docs_assets/logos/bizhawk.png deleted file mode 100644 index 511597b2..00000000 Binary files a/docs_assets/logos/bizhawk.png and /dev/null differ diff --git a/docs_assets/logos/bizhawk.webp b/docs_assets/logos/bizhawk.webp new file mode 100644 index 00000000..da5ad034 Binary files /dev/null and b/docs_assets/logos/bizhawk.webp differ diff --git a/docs_assets/logos/emudeck.png b/docs_assets/logos/emudeck.png deleted file mode 100644 index 408a9415..00000000 Binary files a/docs_assets/logos/emudeck.png and /dev/null differ diff --git a/docs_assets/logos/emudeck.webp b/docs_assets/logos/emudeck.webp new file mode 100644 index 00000000..011379cf Binary files /dev/null and b/docs_assets/logos/emudeck.webp differ diff --git a/docs_assets/logos/emuelec.png b/docs_assets/logos/emuelec.png deleted file mode 100644 index 1c432fc5..00000000 Binary files a/docs_assets/logos/emuelec.png and /dev/null differ diff --git a/docs_assets/logos/emuelec.webp b/docs_assets/logos/emuelec.webp new file mode 100644 index 00000000..1201668d Binary files /dev/null and b/docs_assets/logos/emuelec.webp differ diff --git a/docs_assets/logos/knulli.png b/docs_assets/logos/knulli.png deleted file mode 100644 index fc8885f1..00000000 Binary files a/docs_assets/logos/knulli.png and /dev/null differ diff --git a/docs_assets/logos/knulli.webp b/docs_assets/logos/knulli.webp new file mode 100644 index 00000000..82c85999 Binary files /dev/null and b/docs_assets/logos/knulli.webp differ diff --git a/docs_assets/logos/minui.png b/docs_assets/logos/minui.png deleted file mode 100644 index cc59063a..00000000 Binary files a/docs_assets/logos/minui.png and /dev/null differ diff --git a/docs_assets/logos/minui.webp b/docs_assets/logos/minui.webp new file mode 100644 index 00000000..94ea7981 Binary files /dev/null and b/docs_assets/logos/minui.webp differ diff --git a/docs_assets/logos/misterfpga.png b/docs_assets/logos/misterfpga.png deleted file mode 100644 index 35821644..00000000 Binary files a/docs_assets/logos/misterfpga.png and /dev/null differ diff --git a/docs_assets/logos/misterfpga.webp b/docs_assets/logos/misterfpga.webp new file mode 100644 index 00000000..45c44e4a Binary files /dev/null and b/docs_assets/logos/misterfpga.webp differ diff --git a/docs_assets/logos/nextui.png b/docs_assets/logos/nextui.png deleted file mode 100644 index dd758331..00000000 Binary files a/docs_assets/logos/nextui.png and /dev/null differ diff --git a/docs_assets/logos/nextui.webp b/docs_assets/logos/nextui.webp new file mode 100644 index 00000000..8c5968a0 Binary files /dev/null and b/docs_assets/logos/nextui.webp differ diff --git a/docs_assets/logos/retrobat.png b/docs_assets/logos/retrobat.png deleted file mode 100644 index 2b4395ac..00000000 Binary files a/docs_assets/logos/retrobat.png and /dev/null differ diff --git a/docs_assets/logos/retrobat.webp b/docs_assets/logos/retrobat.webp new file mode 100644 index 00000000..b086f4f6 Binary files /dev/null and b/docs_assets/logos/retrobat.webp differ diff --git a/docs_assets/logos/retropie.png b/docs_assets/logos/retropie.png deleted file mode 100644 index e3885625..00000000 Binary files a/docs_assets/logos/retropie.png and /dev/null differ diff --git a/docs_assets/logos/retropie.webp b/docs_assets/logos/retropie.webp new file mode 100644 index 00000000..70328f9b Binary files /dev/null and b/docs_assets/logos/retropie.webp differ diff --git a/docs_assets/logos/rocknix.png b/docs_assets/logos/rocknix.png deleted file mode 100644 index 8158615d..00000000 Binary files a/docs_assets/logos/rocknix.png and /dev/null differ diff --git a/docs_assets/logos/rocknix.webp b/docs_assets/logos/rocknix.webp new file mode 100644 index 00000000..b75370ee Binary files /dev/null and b/docs_assets/logos/rocknix.webp differ diff --git a/docs_assets/logos/romm.png b/docs_assets/logos/romm.png deleted file mode 100644 index 71d38413..00000000 Binary files a/docs_assets/logos/romm.png and /dev/null differ diff --git a/docs_assets/logos/romm.webp b/docs_assets/logos/romm.webp new file mode 100644 index 00000000..0a2fafaf Binary files /dev/null and b/docs_assets/logos/romm.webp differ diff --git a/docs_assets/logos/sources.json b/docs_assets/logos/sources.json index 590c551d..c5a3f527 100644 --- a/docs_assets/logos/sources.json +++ b/docs_assets/logos/sources.json @@ -2,7 +2,7 @@ "note": "Each platform's own logo, owned by its project and shown only to name that platform. Copied once by scripts/fetch_site_images.py.", "images": [ { - "file": "amberelec.png", + "file": "amberelec.webp", "platform": "amberelec", "source": "https://avatars.githubusercontent.com/u/102820909", "owner": "AmberELEC", @@ -16,28 +16,28 @@ "licence": "repository licence MIT" }, { - "file": "bizhawk.png", + "file": "bizhawk.webp", "platform": "bizhawk", "source": "https://avatars.githubusercontent.com/u/11743303", "owner": "TASVideos", "licence": "GitHub organisation avatar of TASEmulators" }, { - "file": "emudeck.png", + "file": "emudeck.webp", "platform": "emudeck", "source": "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/icons/EmuDeck.png", "owner": "EmuDeck", "licence": "repository licence GPL-3.0" }, { - "file": "emuelec.png", + "file": "emuelec.webp", "platform": "emuelec", "source": "https://avatars.githubusercontent.com/u/57574835", "owner": "EmuELEC", "licence": "GitHub organisation avatar of EmuELEC" }, { - "file": "knulli.png", + "file": "knulli.webp", "platform": "knulli", "source": "https://avatars.githubusercontent.com/u/166268572", "owner": "KNULLI", @@ -51,21 +51,21 @@ "licence": "file from libretro/retroarch-assets, CC-BY-4.0" }, { - "file": "minui.png", + "file": "minui.webp", "platform": "minui", "source": "https://avatars.githubusercontent.com/u/225589", "owner": "Shaun Inman", "licence": "GitHub avatar of shauninman, the author of MinUI" }, { - "file": "misterfpga.png", + "file": "misterfpga.webp", "platform": "misterfpga", "source": "https://avatars.githubusercontent.com/u/29345272", "owner": "MiSTer FPGA", "licence": "GitHub organisation avatar of MiSTer-devel" }, { - "file": "nextui.png", + "file": "nextui.webp", "platform": "nextui", "source": "https://avatars.githubusercontent.com/u/202386615", "owner": "LoveRetro", @@ -86,7 +86,7 @@ "licence": "repository licence GPL-3.0" }, { - "file": "retrobat.png", + "file": "retrobat.webp", "platform": "retrobat", "source": "https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png", "owner": "RetroBat", @@ -100,21 +100,21 @@ "licence": "repository licence GPL-3.0" }, { - "file": "retropie.png", + "file": "retropie.webp", "platform": "retropie", "source": "https://avatars.githubusercontent.com/u/11378204", "owner": "RetroPie", "licence": "GitHub organisation avatar of RetroPie" }, { - "file": "rocknix.png", + "file": "rocknix.webp", "platform": "rocknix", "source": "https://avatars.githubusercontent.com/u/163176886", "owner": "ROCKNIX", "licence": "GitHub organisation avatar of ROCKNIX" }, { - "file": "romm.png", + "file": "romm.webp", "platform": "romm", "source": "https://avatars.githubusercontent.com/u/164058018", "owner": "RomM", diff --git a/scripts/fetch_site_images.py b/scripts/fetch_site_images.py index 49ae239b..783f911c 100644 --- a/scripts/fetch_site_images.py +++ b/scripts/fetch_site_images.py @@ -8,8 +8,9 @@ into ``docs_assets/logos/`` and ``docs_assets/system-icons/``, which are committed, and records where each came from in a ``sources.json`` beside them. The site build only reads those copies (``sitediagrams.localize_images``). -Raster images are normalised to a square PNG (96 px for logos, 48 px for -system icons) so every tile has the same weight, and SVG is kept as published. +Raster logos are normalised to an 80 px square WebP, twice the 40 px they +are shown at, and system icons to a 48 px PNG, so every tile has the same +weight. SVG is kept as published. Normalising needs Pillow. This is a maintenance tool run by hand when a logo changes, not a build step: the build stays on the standard library and PyYAML. @@ -36,7 +37,7 @@ sys.path.insert(0, str(ROOT / "scripts")) LOGO_DIR = ROOT / "docs_assets" / "logos" ICON_DIR = ROOT / "docs_assets" / "system-icons" -LOGO_SIZE = 96 +LOGO_SIZE = 80 ICON_SIZE = 48 AVATAR_HOST = "avatars.githubusercontent.com" USER_AGENT = "retrobios-site-images" @@ -47,7 +48,7 @@ def fetch(url: str) -> bytes: return response.read() -def square_png(data: bytes, size: int) -> bytes: +def square_image(data: bytes, size: int, kind: str = "PNG") -> bytes: try: from PIL import Image except ImportError as exc: @@ -57,7 +58,11 @@ def square_png(data: bytes, size: int) -> bytes: tile = Image.new("RGBA", (size, size), (0, 0, 0, 0)) tile.paste(image, ((size - image.width) // 2, (size - image.height) // 2), image) out = io.BytesIO() - tile.save(out, "PNG", optimize=True) + if kind == "WEBP": + # Quality 90: half the bytes of lossless, edges still clean at 40 px. + tile.save(out, "WEBP", quality=90, method=6) + else: + tile.save(out, "PNG", optimize=True) return out.getvalue() @@ -98,7 +103,7 @@ def fetch_logos() -> None: if is_svg(data): file_name, payload = f"{name}.svg", safe_svg(data, url) else: - file_name, payload = f"{name}.png", square_png(data, LOGO_SIZE) + file_name, payload = f"{name}.webp", square_image(data, LOGO_SIZE, "WEBP") for stale in LOGO_DIR.glob(f"{name}.*"): if stale.name != file_name: stale.unlink() @@ -141,7 +146,7 @@ def fetch_icons() -> None: continue raise file_name = f"{name}.png" - (ICON_DIR / file_name).write_bytes(square_png(data, ICON_SIZE)) + (ICON_DIR / file_name).write_bytes(square_image(data, ICON_SIZE)) entries.append({"file": file_name, "source": url}) print(f" {file_name}") write_manifest( diff --git a/tests/test_site_r2_components.py b/tests/test_site_r2_components.py index 6aad12ed..449e2142 100644 --- a/tests/test_site_r2_components.py +++ b/tests/test_site_r2_components.py @@ -84,8 +84,10 @@ class TablesStackWhenBuilt(unittest.TestCase): "action": {"labels": ["A", "B", "C"], "rows": 40, "action": True}, "pair": {"labels": ["Where", "BIOS folder"], "rows": 3, "action": False}, "short-start": {"labels": ["System", "Runs", "BIOS folder"], "rows": 8, "action": False, "level": "start"}, - "long-setup": {"labels": ["System", "Files", "Status", "Emulators"], "rows": 97, "action": False, "level": "setup"}, + "long-setup": {"labels": ["System", "Files", "Status"], "rows": 97, "action": False, "level": "setup"}, "reference": {"labels": ["A", "B", "C"], "rows": 5, "action": False, "level": "reference"}, + # Four columns do not fit a phone column, on any page (round 3). + "wide": {"labels": ["File", "Region", "Required", "Hash"], "rows": 105, "action": False, "level": "reference"}, } script = ( f"import {{ stackKind }} from {json.dumps(PLUGIN.as_uri())};" @@ -98,7 +100,7 @@ class TablesStackWhenBuilt(unittest.TestCase): ) self.assertEqual(json.loads(result.stdout), { "zip-header": "true", "action": "true", "pair": "kv", - "short-start": "true", "long-setup": None, "reference": None, + "short-start": "true", "long-setup": None, "reference": None, "wide": "true", }) @@ -142,7 +144,7 @@ class SearchAnswersAHashAndShowsNoFileDump(unittest.TestCase): self.assertIn("Look up this hash", source) self.assertIn("files/?hash=${value}", source) self.assertIn("endsWith('#_top')", source) - self.assertIn("tag.dataset.pagefindUiMeta === 'files'", source) + self.assertIn("UNSHOWN_FIELDS.has(tag.dataset.pagefindUiMeta ?? '')", source) # Pagefind's display rules beat the hidden attribute without this. self.assertIn("#starlight__search [hidden] { display: none !important; }", _read(COMPONENTS / "Header.astro")) @@ -167,8 +169,8 @@ def _title_block(html: str) -> str: class LevelBadgeOnlyWhereItAddsSomething(unittest.TestCase): def test_badge_follows_the_crumb(self): - # A tab root has no crumb and keeps its badge. - self.assertIn('class="rb-level rb-level-setup', _title_block(_built("platforms"))) + # A tab root: the header tab already says it (round 3, 2.1). + self.assertNotIn('class="rb-level ', _title_block(_built("platforms"))) # Under the "Start here" or "Your setup" crumb the badge repeated it. for page in ("which-pack", "platforms/batocera"): with self.subTest(page=page): @@ -181,7 +183,7 @@ class TitleLogo(unittest.TestCase): def test_front_matter_field(self): self.assertIn("logo: z.string().optional()", _read(WEBSITE / "src" / "content.config.ts")) title = _read(COMPONENTS / "PageTitle.astro") - self.assertIn('', title) + self.assertIn('', title) class FooterSaysWhoAndWhen(unittest.TestCase): @@ -200,10 +202,10 @@ class FooterSaysWhoAndWhen(unittest.TestCase): class FindAFileGivesAVerdict(unittest.TestCase): def test_lookup_renderer(self): source = _read(SITE_TS) - self.assertIn("', held in the collection at '", source) + self.assertIn("'Held in the collection at '", source) self.assertIn("['Setup', 'Name as declared', 'Destination', 'Hash matched', 'Required']", source) self.assertIn("api/v1/platforms/${item.owner}.json", source) - self.assertIn("api/v1/systems/${slug}.json", source) + self.assertIn("api/v1/systems/${words.slice(0, size).join('-')}.json", source) style = _read(COMPONENTS / "MarkdownContent.astro") self.assertIn("max-width: 40rem;", style) self.assertIn("width: 100%;", style) diff --git a/tests/test_site_r2_gate.py b/tests/test_site_r2_gate.py index 3a08cfa8..a83dffa2 100644 --- a/tests/test_site_r2_gate.py +++ b/tests/test_site_r2_gate.py @@ -192,7 +192,7 @@ class TheIntegrationItems(unittest.TestCase): with self.subTest(platform=key): block = re.search(r'
.*?
', html, re.S) self.assertIsNotNone(block) - self.assertRegex(block[0], r' str: + return path.read_text(encoding="utf-8") + + +def _built(page: str) -> str: + path = DIST / page / "index.html" + if not path.is_file(): + raise unittest.SkipTest(f"no built site at {path}") + return _read(path) + + +def _title_block(html: str) -> str: + """What PageTitle draws: from the main column to the end of the h1.""" + end = html.index("", html.index('id="_top"')) + return html[html.rfind(" list[str]: + return re.findall(r"", html, re.S) + + +def _headers(table: str) -> list[str]: + head = re.search(r"(.*?)", table, re.S) + cells = re.findall(r"]*>(.*?)", head.group(1), re.S) if head else [] + return [re.sub(r"<[^>]+>", "", cell).strip() for cell in cells] + + +def _node_module(script: str) -> str: + node = shutil.which("node") + if not node: + raise unittest.SkipTest("node is missing") + result = subprocess.run( + [node, "--input-type=module-typescript", "-e", script], + capture_output=True, text=True, check=True, cwd=WEBSITE, + ) + return result.stdout + + +def _webp_size(data: bytes) -> tuple[int, int]: + """Width and height from a WebP header (VP8, VP8L or VP8X).""" + if data[:4] != b"RIFF" or data[8:12] != b"WEBP": + raise ValueError("not a WebP file") + chunk = data[12:16] + if chunk == b"VP8 ": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + if chunk == b"VP8X": + return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1 + raise ValueError(f"unknown WebP chunk {chunk!r}") + + +class TitleLogoOnAPlate(unittest.TestCase): + """The RetroArch and Batocera marks are black: beside the title they + vanished on the dark theme, and the image had no width (Lighthouse + unsized-images, V3, T-F3).""" + + def test_the_component_draws_the_tile_square(self): + title = _read(COMPONENTS / "PageTitle.astro") + self.assertIn('width={MARK} height={MARK}', title) + self.assertIn("const MARK = 40;", title) + style = title[title.index(".rb-title-mark {"):] + style = style[:style.index("}")] + for rule in ("width: 40px;", "height: 40px;", "padding: 3px;", "border: 1px solid var(--rb-border);", + "border-radius: 6px;", "background: #fff;", "object-fit: contain;"): + with self.subTest(rule=rule): + self.assertIn(rule, style) + + def test_every_setup_page_has_a_sized_mark(self): + index = json.loads(_read(WEBSITE / "public" / "data" / "picker" / "index.json")) + for entry in index: + with self.subTest(setup=entry["platform"]): + block = _title_block(_built(f"platforms/{entry['platform']}")) + self.assertIn('class="rb-title-mark', block) + for image in re.findall(r"]*>", block): + self.assertIn('width="40"', image) + self.assertIn('height="40"', image) + + def test_a_setup_with_no_logo_gets_its_initials(self): + block = _title_block(_built("platforms/myminui")) + self.assertNotIn("]*>.*?>MM") + + +class NoLevelBadgeOnATabRoot(unittest.TestCase): + def test_tab_roots(self): + for page in TAB_ROOTS: + with self.subTest(page=page or "home"): + self.assertNotIn('class="rb-level ', _title_block(_built(page))) + + +class FooterLicenceMatchesTheLicenceFile(unittest.TestCase): + """The footer said MIT covers the tooling. LICENSE names the metadata too.""" + + def test_footer(self): + self.assertIn("metadata", _read(REPO_ROOT / "LICENSE").split("MIT License")[0]) + html = _built("data") + footer = re.sub(r"<[^>]+>", "", html[html.index('class="rb-footer'):]) + self.assertIn("MIT for the tooling and metadata.", footer) + + +class WideTablesStackAndFiltersAreBuilt(unittest.TestCase): + """Level 2 and 3 tables overflowed a phone (emulator summary, gaps, + verification modes, emulators index, data: V5, J8). The filter box, + added after paint, pushed the table down: CLS 0.066 on + gaps-files-to-source (T-F10).""" + + def test_every_wide_table_stacks(self): + for page in ("emulators/duckstation", "gaps", "gaps-files-to-source", "emulators", "data", + "wiki/verification-modes"): + html = (DIST / page / "index.html") + if not html.is_file(): + continue + for table in _tables(_read(html)): + if len(_headers(table)) < 4: + continue + with self.subTest(page=page, header=_headers(table)): + self.assertIn('data-rb-stack="true"', table.split(">", 1)[0]) + + def test_long_tables_get_their_labels_at_runtime(self): + """Labels on every cell of a long stacked table were 122 KB of + emulators/mame and took it over the 600,000-byte page budget.""" + self.assertIn("if (kind === 'kv' || rows.length >= RECORD_ROWS) return;", _read(PLUGIN)) + source = _read(SITE_TS) + self.assertIn("const labelCells = (table: HTMLTableElement) => {", source) + self.assertIn("labelCells(table);", source) + html = _built("emulators/mame") + self.assertLess(len(html.encode()), 600_000) + + def test_the_filter_box_is_in_the_html(self): + html = _built("gaps-files-to-source") + boxes = re.findall(r'
(.*?)
\s*Filter [^<]+') + self.assertRegex(box, r'[\d,]+ rows') + source = _read(SITE_TS) + self.assertNotIn("controls.className = 'rb-table-filter'", source) + self.assertIn("controls?.matches('.rb-table-filter')", source) + + def test_every_long_table_has_its_box(self): + html = _built("systems") + for table in _tables(html): + rows = len(re.findall(r"", 1)[-1])) + if rows < 20: + continue + start = html.index(table) + self.assertIn('class="rb-table-filter"', html[max(0, start - 600):start]) + + def test_the_placeholder_names_the_columns(self): + """Every filter offered "Name, platform, system, hash...", over the + console list too, which has no hash (V small).""" + self.assertNotIn("Name, platform, system, hash", _read(SITE_TS)) + cases = [["File", "Region", "Required", "Hash"], ["Console", "Files"], ["System", "MD5", "Size"]] + out = _node_module( + f"import {{ filterPlaceholder }} from {json.dumps(PLUGIN.as_uri())};" + f"console.log(JSON.stringify({json.dumps(cases)}.map(filterPlaceholder)));" + ) + self.assertEqual(json.loads(out), ["File, region, required, hash...", "Console, files...", "System, MD5, size..."]) + html = _built("systems") + self.assertRegex(html, r'placeholder="Console[^"]*\.\.\."') + self.assertNotRegex(html, r'placeholder="[^"]*hash[^"]*"') + + +class HashLookupGivesOneVerdict(unittest.TestCase): + """The lookup read "24 matches", "Setups that list it (12)" over 17 rows, + and emulators by id. ?md5= showed the empty form (N8, J7).""" + + def test_parameters(self): + source = _read(SITE_TS) + self.assertIn("const HASH_PARAMS = ['hash', 'md5', 'crc32', 'sha256'];", source) + self.assertIn("HASH_PARAMS.map((key) => params.get(key))", source) + + def test_counts_say_what_they_count(self): + source = _read(SITE_TS) + self.assertNotIn("'match' : 'matches'", source) + self.assertIn("count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}", source) + self.assertIn("count(setups.size, 'setup lists', 'setups list')", source) + self.assertIn("'file in the collection', 'files in the collection'", source) + + def test_emulators_by_name(self): + source = _read(SITE_TS) + self.assertIn("json(`api/v1/emulators/${owner}.json`)", source) + self.assertIn("profile.emulator || profile.display_name || owner", source) + profile = json.loads(_read(DIST / "api" / "v1" / "emulators" / "duckstation.json")) \ + if (DIST / "api" / "v1" / "emulators" / "duckstation.json").is_file() else None + if profile is None: + self.skipTest("no built site") + self.assertEqual(profile["items"][0]["profile"]["emulator"], "DuckStation") + + def test_a_console_page_card_reads_its_manufacturer_file(self): + """Every collection match on a console page (systems/sony-playstation/) + asked for api/v1/systems/sony-playstation.json, which does not exist: + the verdict lost the file's path.""" + source = _read(SITE_TS) + self.assertIn("api/v1/systems/${words.slice(0, size).join('-')}.json", source) + api = DIST / "api" / "v1" / "systems" + if not api.is_dir(): + self.skipTest("no built site") + self.assertFalse((api / "sony-playstation.json").exists()) + self.assertTrue((api / "sony.json").is_file()) + + def test_no_match_leads_somewhere(self): + source = _read(SITE_TS) + self.assertIn("link('contributing/', 'add it')", source) + anchor = re.search(r"const WANTED = 'provenance/#([^']+)';", source) + self.assertIsNotNone(anchor) + self.assertIn(f'id="{anchor[1]}"', _built("provenance")) + self.assertNotIn("No file the site knows has this hash.", source) + + +class HashAFileHere(unittest.TestCase): + """Find a file asked for a hash only: hashing meant another tool (J7).""" + + def test_the_chooser(self): + source = _read(SITE_TS) + self.assertIn("input.type = 'file';", source) + self.assertIn("It is hashed on this device. Nothing is uploaded.", source) + self.assertIn("crypto.subtle.digest(name, buffer)", source) + self.assertIn("['SHA-1', 'SHA-256']", source) + self.assertIn("lookUp(status, result, [found.sha1, found.md5, found.crc32, found.sha256])", source) + + def test_md5_and_crc32_match_python(self): + source = _read(SITE_TS) + block = source[source.index("// Hashing a file on this device"):source.index("type FileHashes")] + sizes = [0, 1, 55, 56, 63, 64, 65, 119, 120, 128, 1000, 70_001] + out = _node_module(block + ( + f"console.log(JSON.stringify({json.dumps(sizes)}.map((n) => {{" + "const bytes = new Uint8Array(n);" + "for (let i = 0; i < n; i += 1) bytes[i] = (i * 31 + 7) & 0xff;" + "return [md5(bytes), crc32(bytes)]; })));" + )) + for size, (md5, crc) in zip(sizes, json.loads(out)): + data = bytes((i * 31 + 7) & 0xFF for i in range(size)) + with self.subTest(size=size): + self.assertEqual(md5, hashlib.md5(data).hexdigest()) + self.assertEqual(crc, f"{zlib.crc32(data):08x}") + + +class SearchFoldsPagesUnderAHash(unittest.TestCase): + """Under "Look up this hash" came 14 unrelated pages: Magnavox for + E490.WAV, Philips for cdi490a.zip (N8).""" + + def test_fold(self): + source = _read(SITE_TS) + self.assertIn("toggle.textContent = 'Pages that mention these characters';", source) + self.assertIn("toggle.setAttribute('aria-expanded', 'false');", source) + header = _read(COMPONENTS / "Header.astro") + self.assertIn( + ".pagefind-ui__drawer[data-rb-hash]:not([data-rb-pages-shown]) .pagefind-ui__results-area { display: none; }", + header, + ) + + def test_search_fields_are_weighted_and_never_shown(self): + config = _read(CONFIG) + self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5 },", config) + self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices']);", _read(SITE_TS)) + + +class OsTabsKeepTheChoiceInView(unittest.TestCase): + def test_the_row_scrolls_not_the_page(self): + source = _read(SITE_TS) + self.assertIn("list.scrollLeft", source) + self.assertNotIn("scrollIntoView", source) + self.assertIn("osTabs();", source) + + +class CommandsNeverBreakInsideAWord(unittest.TestCase): + """`--` / `platform retroarch` at 1440 px, `--` / `system` and + `nintendo-` / `64dd` on a phone: a browser breaks a line after a hyphen, + whatever overflow-wrap says.""" + + def test_the_words_are_wrapped_and_the_text_kept(self): + # Shiki cuts `--platform` into two tokens: the span has to hold both. + line = {"type": "element", "tagName": "div", "properties": {"className": ["code"]}, "children": [ + {"type": "element", "tagName": "span", "properties": {"style": "a"}, "children": [ + {"type": "text", "value": "sh -s "}]}, + {"type": "element", "tagName": "span", "properties": {"style": "b"}, "children": [ + {"type": "text", "value": "--"}]}, + {"type": "element", "tagName": "span", "properties": {"style": "c"}, "children": [ + {"type": "text", "value": "platform retroarch --system nintendo-64dd"}]}, + ]} + out = json.loads(_node_module( + f"import {{ unbreakWords }} from {json.dumps(UNBROKEN.as_uri())};" + f"console.log(JSON.stringify(unbreakWords({json.dumps(line)})));" + )) + + def text(node: dict) -> str: + return node.get("value", "") + "".join(text(child) for child in node.get("children", [])) + + self.assertEqual("".join(text(n) for n in out), "sh -s --platform retroarch --system nintendo-64dd") + words = [text(n) for n in out if n.get("properties", {}).get("className") == ["rb-nb"]] + self.assertEqual(words, ["-s", "--platform", "--system", "nintendo-64dd"]) + # The colours of each token survive the cut. + platform = next(n for n in out if text(n) == "--platform") + self.assertEqual([c["properties"]["style"] for c in platform["children"]], ["b", "c"]) + + def test_registered(self): + config = _read(CONFIG) + self.assertIn("import unbrokenTokens from './src/plugins/ec-unbroken-tokens.mjs';", config) + self.assertIn("plugins: [unbrokenTokens()],", config) + self.assertIn("baseStyles: `.${CLASS} { white-space: nowrap; }`", _read(UNBROKEN)) + picker = _read(COMPONENTS / "FilePicker.astro") + self.assertIn("showCommand(cmd, `${INSTALL} --platform", picker) + self.assertIn(".rb-picker .rb-pick-cmd .rb-nb { white-space: nowrap; }", picker) + self.assertIn("navigator.clipboard?.writeText(cmd.textContent ?? '')", picker) + + def test_copy_gives_the_rendered_text(self): + """The copy button's text is the block's text: the spans add nothing. + Expressive Code leaves a leading comment line out of the copy.""" + import html as html_lib + + page = _built("platforms/retroarch") + figures = re.findall(r"", page, re.S) + self.assertTrue(figures) + self.assertIn('', page) + for figure in figures: + copied = re.search(r'data-code="([^"]*)"', figure) + if not copied: + continue + lines = re.findall(r'
(.*?)
', figure, re.S) + shown = "\n".join(html_lib.unescape(re.sub(r"<[^>]+>", "", line)) for line in lines) + code = html_lib.unescape(copied[1]).replace("\x7f", "\n") + with self.subTest(code=code[:60]): + self.assertTrue(shown.endswith(code), (shown, code)) + + +class LogosAtTheirDisplaySize(unittest.TestCase): + """136 KiB of the home page was 96 px PNG logos shown at 40 px (T-F7).""" + + def test_raster_logos_are_small_webp(self): + manifest = json.loads(_read(LOGOS / "sources.json"))["images"] + self.assertTrue(manifest) + for entry in manifest: + path = LOGOS / entry["file"] + with self.subTest(file=entry["file"]): + self.assertTrue(path.is_file()) + self.assertIn(path.suffix, (".webp", ".svg")) + if path.suffix == ".webp": + self.assertEqual(_webp_size(path.read_bytes()), (LOGO_PIXELS, LOGO_PIXELS)) + self.assertEqual(sorted(p.name for p in LOGOS.glob("*.png")), []) + script = _read(REPO_ROOT / "scripts" / "fetch_site_images.py") + self.assertIn(f"LOGO_SIZE = {LOGO_PIXELS}", script) + self.assertIn('square_image(data, LOGO_SIZE, "WEBP")', script) + + def test_pages_point_at_the_copies(self): + html = _built("platforms/emudeck") + self.assertIn('src="/retrobios/assets/logos/emudeck.webp"', html) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/astro.config.mjs b/website/astro.config.mjs index cb2996f8..71fa2b65 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -13,6 +13,7 @@ import starlightLinksValidator from 'starlight-links-validator'; import sidebar from './src/generated/sidebar.json' with { type: 'json' }; import lastmod from './src/generated/lastmod.json' with { type: 'json' }; import stackTables from './src/plugins/rehype-stack-tables.mjs'; +import unbrokenTokens from './src/plugins/ec-unbroken-tokens.mjs'; const SITE = 'https://abdess.github.io'; const BASE = '/retrobios/'; @@ -84,11 +85,14 @@ export default defineConfig({ routeMiddleware: ['./src/routeData.ts'], tableOfContents: { minHeadingLevel: 2, maxHeadingLevel: 3 }, // Commands wrap instead of scrolling, so the end of a one-liner stays - // visible. A command block is titled (Terminal, PowerShell) and drawn - // without the three window dots. Starlight writes the dots into each - // theme after the global style overrides, so they are set here. + // visible, and never inside a word that holds a hyphen (`--platform`, + // `nintendo-64dd`: src/plugins/ec-unbroken-tokens.mjs). A command block + // is titled (Terminal, PowerShell) and drawn without the three window + // dots. Starlight writes the dots into each theme after the global + // style overrides, so they are set here. expressiveCode: { defaultProps: { wrap: true }, + plugins: [unbrokenTokens()], customizeTheme(theme) { theme.styleOverrides.frames = { ...theme.styleOverrides.frames, terminalTitlebarDotsOpacity: '0' }; return theme; @@ -102,14 +106,18 @@ export default defineConfig({ // ten times its text, so "dolphin" finds Dolphin, and the file names of // a system page (its `files` field, written by sitestarlight.py) count // half, so "scph5501" finds the Sony page without a file name winning - // every other query. + // every other query. Two fields name the page's subject in other words + // and weigh half a title: `aliases`, the ids setups file a system under + // (psx, dreamcast, gba), and `devices`, the handhelds and boxes a + // setup's own site names (R36S, Miyoo Mini). All three are attributes + // on an empty element, so no text of theirs reaches an excerpt. pagefind: { ranking: { pageLength: 0.1, termFrequency: 0.5, termSaturation: 1.4, termSimilarity: 9, - metaWeights: { title: 10, files: 0.5 }, + metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5 }, }, }, plugins: [ diff --git a/website/src/components/FilePicker.astro b/website/src/components/FilePicker.astro index 6444d7a4..1c09e871 100644 --- a/website/src/components/FilePicker.astro +++ b/website/src/components/FilePicker.astro @@ -66,6 +66,25 @@ const packs = Object.fromEntries( const INSTALL = 'curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s --'; const MAX_ROWS = 500; + // A word that holds a hyphen (--platform, nintendo-64dd) never breaks + // inside: the same rule as the command blocks (ec-unbroken-tokens.mjs). + // Only spans are added, the text copied is the command itself. + const UNBROKEN = /(? { + const nodes: (Node | string)[] = []; + let last = 0; + for (const match of command.matchAll(UNBROKEN)) { + nodes.push(command.slice(last, match.index)); + const word = document.createElement('span'); + word.className = 'rb-nb'; + word.textContent = match[0]; + nodes.push(word); + last = match.index! + match[0].length; + } + nodes.push(command.slice(last)); + target.replaceChildren(...nodes); + }; const size = (bytes: number) => bytes >= 1024 ** 3 ? `${(bytes / 1024 ** 3).toFixed(1)} GB` @@ -107,7 +126,7 @@ const packs = Object.fromEntries( ? `${count(files.length)} files, ${size(total)}, for ${count(picked.length)} of ${count(data.systems.length + (data.other ? 1 : 0))} choices. Folder: ${data.folder || 'the setup BIOS folder'}.` : 'Choose at least one system.'; output.hidden = !picked.length; - cmd.textContent = `${INSTALL} --platform ${data.platform}${all ? '' : ` --system ${picked.join(',')}`}`; + showCommand(cmd, `${INSTALL} --platform ${data.platform}${all ? '' : ` --system ${picked.join(',')}`}`); tbody.replaceChildren(...files.slice(0, MAX_ROWS).map(([dest, bytes]) => { const row = document.createElement('tr'); const where = document.createElement('td'); @@ -221,7 +240,8 @@ const packs = Object.fromEntries( .rb-picker .rb-pick-systems-head { display: flex; gap: 0.5rem; align-items: center; } .rb-picker .rb-pick-systems-head > * { margin: 0; } .rb-picker .rb-pick-filter { width: 100%; margin: 0.5rem 0; } - .rb-picker .rb-pick-cmd code { white-space: pre-wrap; overflow-wrap: anywhere; } + .rb-picker .rb-pick-cmd code { white-space: pre-wrap; overflow-wrap: break-word; } + .rb-picker .rb-pick-cmd .rb-nb { white-space: nowrap; } .rb-picker .rb-pick-result { margin-top: 1rem; font-weight: 600; } .rb-picker fieldset { flex-direction: column; diff --git a/website/src/components/Footer.astro b/website/src/components/Footer.astro index a038c1c6..e96db4d8 100644 --- a/website/src/components/Footer.astro +++ b/website/src/components/Footer.astro @@ -29,7 +29,7 @@ const date = releaseDate(tag); Packs from release {tag}{date && <>, }.

- MIT for the tooling. BIOS and firmware files are third-party system software, preserved for personal backup, + MIT for the tooling and metadata. BIOS and firmware files are third-party system software, preserved for personal backup, archival and interoperability.

diff --git a/website/src/components/Header.astro b/website/src/components/Header.astro index 271f7904..b8d5299b 100644 --- a/website/src/components/Header.astro +++ b/website/src/components/Header.astro @@ -38,7 +38,24 @@ const topics = Astro.locals.starlightSidebarTopics?.topics ?? []; border-radius: 0.5rem; overflow-wrap: anywhere; } + .rb-search-hash p { margin: 0; } .rb-search-hash a { font-weight: 600; } + /* Under a hash, the pages that only mention its characters stay folded. */ + .pagefind-ui__drawer[data-rb-hash]:not([data-rb-pages-shown]) .pagefind-ui__results-area { display: none; } + .rb-search-hash .rb-search-pages { + min-height: 24px; + margin-top: 0.5rem; + padding: 0; + border: 0; + background: none; + color: var(--sl-color-gray-2); + font: inherit; + font-size: var(--sl-text-sm); + text-decoration: underline; + cursor: pointer; + } + .rb-search-hash .rb-search-pages::before { content: '+ '; } + .rb-search-hash .rb-search-pages[aria-expanded='true']::before { content: '- '; } /* Pagefind's own display rules would show what site.ts hides. */ #starlight__search [hidden] { display: none !important; } diff --git a/website/src/components/MarkdownContent.astro b/website/src/components/MarkdownContent.astro index 402cedf4..bb507b3b 100644 --- a/website/src/components/MarkdownContent.astro +++ b/website/src/components/MarkdownContent.astro @@ -30,4 +30,25 @@ const { searchWeight } = Astro.locals.starlightRoute.entry.data; max-width: 40rem; box-sizing: border-box; } + /* The file chooser site.ts adds under the form, and the four hashes it + shows once the file is read. */ + .sl-markdown-content .rb-file-pick { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.35rem; margin-top: 1rem; } + .sl-markdown-content .rb-file-pick > * { margin: 0; } + .sl-markdown-content .rb-file-pick label { font-weight: 600; } + .sl-markdown-content .rb-file-pick input { width: 100%; min-width: 0; font: inherit; } + .sl-markdown-content .rb-file-pick input::file-selector-button { + min-height: 44px; + margin-inline-end: 0.75rem; + padding: 0 1rem; + border: 1px solid var(--rb-border-strong); + border-radius: 4px; + background: var(--sl-color-bg); + color: var(--sl-color-text); + font: inherit; + font-weight: 600; + cursor: pointer; + } + .sl-markdown-content .rb-file-pick-note { color: var(--sl-color-gray-3); font-size: var(--sl-text-sm); } + .sl-markdown-content .rb-file-pick-hashes code { overflow-wrap: anywhere; } + .sl-markdown-content .rb-file-pick-hashes[hidden] { display: none; } diff --git a/website/src/components/PageTitle.astro b/website/src/components/PageTitle.astro index c0dd8d4d..168949b3 100644 --- a/website/src/components/PageTitle.astro +++ b/website/src/components/PageTitle.astro @@ -1,9 +1,16 @@ --- // Breadcrumbs, the level label and the title. The trail says which tab a -// deep page belongs to; the label says who the page is written for, and is -// left out where the first crumb already names that tab. A page with a -// `logo` (a path under the site root) shows it beside its title. +// deep page belongs to, and the label says who the page is written for. The +// label is left out where the header tab already says it: on a tab's own +// page, and where the first crumb names that tab. A page with a `logo` (a +// path under the site root) shows it beside its title, on a white square so +// a dark mark stays visible in the dark theme. A setup page with no logo file +// gets its initials in the same square, as its tile does (sitemarkup.py +// platform_tiles). +import setups from '../../public/data/picker/index.json'; + const PAGE_TITLE_ID = '_top'; +const MARK = 40; const LABELS = { start: 'Start here', @@ -22,11 +29,19 @@ const TABS = { contribute: 'Contribute', } as const; -const { title, level, logo } = Astro.locals.starlightRoute.entry.data; +// Up to two capitals of a name: MyMinUI is MM (sitemarkup.initials). +const initials = (name: string): string => { + const capitals = [...name].filter((c) => c !== c.toLowerCase()); + return capitals.slice(0, 2).join('') || name.slice(0, 2).toUpperCase(); +}; + +const { id, data } = Astro.locals.starlightRoute.entry; +const { title, level, logo } = data; const crumbs = Astro.locals.rbBreadcrumbs ?? []; -const badge = level && crumbs[0]?.label !== TABS[level] && crumbs[0]?.label !== LABELS[level]; +const badge = level && crumbs.length > 0 && crumbs[0].label !== TABS[level] && crumbs[0].label !== LABELS[level]; const base = import.meta.env.BASE_URL.replace(/\/?$/, '/'); const logoSrc = logo && (/^(https?:)?\//.test(logo) ? logo : base + logo); +const setup = !logoSrc && setups.find((entry) => id === `platforms/${entry.platform}`); --- { @@ -42,9 +57,17 @@ const logoSrc = logo && (/^(https?:)?\//.test(logo) ? logo : base + logo); } {badge && {LABELS[level]}} { - logoSrc ? ( + logoSrc || setup ? ( ) : ( @@ -67,11 +90,16 @@ const logoSrc = logo && (/^(https?:)?\//.test(logo) ? logo : base + logo); margin-top: 0.25rem; } .rb-title-logo h1 { margin-top: 0; } - .rb-title-logo img { + /* The tiles' square: white in both themes, a hairline, the mark inside. */ + .rb-title-mark { flex: none; - width: auto; + box-sizing: border-box; + width: 40px; height: 40px; - max-width: 8rem; + padding: 3px; + border: 1px solid var(--rb-border); + border-radius: 6px; + background: #fff; object-fit: contain; } diff --git a/website/src/plugins/ec-unbroken-tokens.mjs b/website/src/plugins/ec-unbroken-tokens.mjs new file mode 100644 index 00000000..ab59a79d --- /dev/null +++ b/website/src/plugins/ec-unbroken-tokens.mjs @@ -0,0 +1,96 @@ +// Command blocks wrap (astro.config.mjs, wrap: true) so the end of a +// one-liner stays visible. A browser may also break a line after a hyphen, +// which split `--platform` into `--` and `platform`, and `nintendo-64dd` +// into `nintendo-` and `64dd`: a reader who types the command reads two +// things. Each word that holds a hyphen is put in a span that does not +// break inside. Only spans are added: the text, the copy button's +// data-code and a selection copied by hand are unchanged. +// +// A word longer than LONGEST is left breakable: an address that long would +// push the block wider than a phone. + +export const LONGEST = 48; +export const UNBROKEN = new RegExp(`(? (node.type === 'text' ? node.value : (node.children ?? []).map(textOf).join('')); + +// A token span holds one text node: it can be cut in two, both halves +// keeping its colours. +const cuttable = (node) => + node.type === 'text' || (node.type === 'element' && node.children?.length === 1 && node.children[0].type === 'text'); + +const slice = (node, from, to) => + node.type === 'text' + ? { type: 'text', value: node.value.slice(from, to) } + : { ...node, properties: { ...node.properties }, children: [{ type: 'text', value: node.children[0].value.slice(from, to) }] }; + +// The children of *container*, with every hyphenated word wrapped in one +// unbreakable span. A word that crosses a node this cannot cut is left as is. +export const unbreakWords = (container) => { + const children = container.children ?? []; + const text = children.map(textOf).join(''); + const ranges = [...text.matchAll(UNBROKEN)].map((match) => [match.index, match.index + match[0].length]); + if (!ranges.length) return children; + const cuts = [...new Set(ranges.flat())].sort((a, b) => a - b); + const pieces = []; + let position = 0; + for (const child of children) { + const end = position + textOf(child).length; + const inner = cuts.filter((cut) => cut > position && cut < end); + if (inner.length && cuttable(child)) { + let start = position; + for (const cut of [...inner, end]) { + pieces.push({ node: slice(child, start - position, cut - position), from: start, to: cut }); + start = cut; + } + } else { + pieces.push({ node: child, from: position, to: end }); + } + position = end; + } + const ends = new Map(ranges); + const out = []; + for (let index = 0; index < pieces.length; ) { + const end = ends.get(pieces[index].from); + if (end !== undefined && pieces[index].to > pieces[index].from) { + let last = index; + while (last < pieces.length && pieces[last].to < end) last += 1; + if (last < pieces.length && pieces[last].to === end) { + out.push({ + type: 'element', + tagName: 'span', + properties: { className: [CLASS] }, + children: pieces.slice(index, last + 1).map((piece) => piece.node), + }); + index = last + 1; + continue; + } + } + out.push(pieces[index].node); + index += 1; + } + return out; +}; + +// The element of a rendered line that holds its tokens (div.code), else +// the line itself. +const tokensOf = (line) => { + for (const child of line.children ?? []) { + if (child.type === 'element' && String(child.properties?.className ?? '').split(/[\s,]+/).includes('code')) return child; + } + return line; +}; + +export default function unbrokenTokens() { + return { + name: 'rb-unbroken-tokens', + baseStyles: `.${CLASS} { white-space: nowrap; }`, + hooks: { + postprocessRenderedLine({ renderData }) { + const container = tokensOf(renderData.lineAst); + container.children = unbreakWords(container); + }, + }, + }; +} diff --git a/website/src/plugins/rehype-stack-tables.mjs b/website/src/plugins/rehype-stack-tables.mjs index 29b9235a..061d6bfa 100644 --- a/website/src/plugins/rehype-stack-tables.mjs +++ b/website/src/plugins/rehype-stack-tables.mjs @@ -1,15 +1,22 @@ -// Tables a phone reads one row at a time, marked when the page is built so -// the stacked layout is there on first paint (the CSS is retrobios.css, -// table[data-rb-stack]). The rule reads the table, never the page that -// wrote it: +// Tables, settled when the page is built so the layout is right on first +// paint (the CSS is retrobios.css, table[data-rb-stack] and .rb-table-filter). +// The rule reads the table, never the page that wrote it. +// +// Which tables a phone reads one row at a time: +// - two columns: a list of label and value, data-rb-stack="kv"; +// - four columns or more: too wide for a phone column, one block per row, +// each cell under its column name; // - a row that ends in an action (a release download, a button, the link to // a setup's install command) or a header that names a ZIP column: one -// block per row, each cell under its column name; -// - two columns: a list of label and value, data-rb-stack="kv"; -// - on a level 0 or 1 page (`level: start` or `setup`), a table short enough -// to read as records (under 20 rows, the size from which site.ts adds a -// filter): one block per row. A longer one is a list scanned by column and -// keeps its scroll. +// block per row, so the action is never behind a horizontal scroll; +// - on a level 0 or 1 page (`level: start` or `setup`), a three-column table +// short enough to read as records (under RECORD_ROWS rows): one block per +// row. A longer one is a list scanned by column and keeps its scroll. +// +// A table of RECORD_ROWS rows or more gets its filter box here, above it. +// Written by site.ts once the page had painted, the box pushed the table +// down (CLS 0.066 on gaps-files-to-source). site.ts only wires it. +// // Satteri runs hast plugins (the rehype stage of its pipeline): this one is // registered in astro.config.mjs through satteri({ hastPlugins }). @@ -17,7 +24,11 @@ const ACTION_HREF = /\/releases\/(download\/|latest\/?$)|#install$/; const ACTION_CLASS = /\bclass=["'][^"']*\brb-button\b/; const DOWNLOAD_HEADER = /^(zip|download)$/i; const READER_LEVELS = new Set(['start', 'setup']); +const HEADING = /^h[1-6]$/; export const RECORD_ROWS = 20; +export const WIDE_COLUMNS = 4; +// The column names a filter's placeholder offers, at most. +const PLACEHOLDER_COLUMNS = 4; const elements = (node, tag, found = []) => { for (const child of node.children ?? []) { @@ -29,23 +40,87 @@ const elements = (node, tag, found = []) => { return found; }; +const classes = (node) => String(node.properties?.className ?? '').split(/[\s,]+/); + const holdsAction = (node) => (node.children ?? []).some( (child) => (child.type === 'element' && ((child.tagName === 'a' && ACTION_HREF.test(String(child.properties?.href ?? ''))) || - String(child.properties?.className ?? '').split(/[\s,]+/).includes('rb-button') || + classes(child).includes('rb-button') || holdsAction(child))) || (child.type === 'raw' && ACTION_CLASS.test(child.value ?? '')), ); export const stackKind = ({ labels, rows, action, level }) => { if (labels.length === 2) return 'kv'; + if (labels.length >= WIDE_COLUMNS) return 'true'; if (action || DOWNLOAD_HEADER.test(labels.at(-1) ?? '')) return 'true'; if (READER_LEVELS.has(level) && labels.length > 2 && rows < RECORD_ROWS) return 'true'; return null; }; +// "File, region, required, hash...": what the rows can be filtered by, in +// the table's own words. A word that is not capitalised prose (MD5, ZIP) +// keeps its case. +export const filterPlaceholder = (labels) => { + const words = labels.filter(Boolean).slice(0, PLACEHOLDER_COLUMNS); + if (!words.length) return 'Type to filter...'; + const lowered = words.map((word, index) => + index > 0 && /^[A-Z][a-z]/.test(word) ? word[0].toLowerCase() + word.slice(1) : word, + ); + return `${lowered.join(', ')}...`; +}; + +// The heading a table sits under: the nearest one before it, looking up +// through the blocks that hold it, else the page title. site.ts names the +// table's scroll region the same way. +const headingBefore = (ctx, table, title) => { + for (let block = table; block; block = ctx.parent(block)) { + const parent = ctx.parent(block); + if (!parent) break; + const siblings = parent.children ?? []; + for (let index = (ctx.indexOf(block) ?? 0) - 1; index >= 0; index -= 1) { + const node = siblings[index]; + if (node.type !== 'element') continue; + const heading = HEADING.test(node.tagName) + ? node + : classes(node).includes('sl-heading-wrapper') + ? (node.children ?? []).find((child) => child.type === 'element' && HEADING.test(child.tagName)) + : undefined; + const text = heading && ctx.textContent(heading).trim(); + if (text) return text; + } + } + return title || 'this table'; +}; + +const filterBox = ({ id, heading, labels, rows }) => ({ + type: 'element', + tagName: 'div', + properties: { className: ['rb-table-filter'] }, + children: [ + { + type: 'element', + tagName: 'label', + properties: { htmlFor: id }, + children: [{ type: 'text', value: `Filter ${heading}` }], + }, + { + type: 'element', + tagName: 'input', + properties: { id, type: 'search', autoComplete: 'off', placeholder: filterPlaceholder(labels) }, + children: [], + }, + { + type: 'element', + tagName: 'span', + properties: { className: ['rb-table-filter-count'], ariaLive: 'polite' }, + children: [{ type: 'text', value: `${rows.toLocaleString('en')} rows` }], + }, + ], +}); + export default function stackTables() { return { name: 'rb-stack-tables', @@ -57,11 +132,19 @@ export default function stackTables() { if (!head || !body) return; const labels = elements(head, 'th').map((cell) => ctx.textContent(cell).trim()); const rows = elements(body, 'tr'); - const level = ctx.data.astro?.frontmatter?.level; - const kind = stackKind({ labels, rows: rows.length, action: holdsAction(body), level }); + const frontmatter = ctx.data.astro?.frontmatter ?? {}; + if (rows.length >= RECORD_ROWS) { + ctx.data.rbFilters = (ctx.data.rbFilters ?? 0) + 1; + const heading = headingBefore(ctx, table, frontmatter.title); + ctx.insertBefore(table, filterBox({ id: `rb-table-filter-${ctx.data.rbFilters}`, heading, labels, rows: rows.length })); + } + const kind = stackKind({ labels, rows: rows.length, action: holdsAction(body), level: frontmatter.level }); if (!kind) return; ctx.setProperty(table, 'dataRbStack', kind); - if (kind === 'kv') return; + // A long table's labels are written by site.ts when it nears the + // screen: on mame they were 122 KB of the page. The label is drawn + // in the room the cell already keeps for it, so nothing moves. + if (kind === 'kv' || rows.length >= RECORD_ROWS) return; for (const row of rows) { elements(row, 'td').forEach((cell, column) => { if (labels[column] && cell.properties?.dataLabel === undefined) ctx.setProperty(cell, 'dataLabel', labels[column]); diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index c503a0ee..07066177 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -1,17 +1,22 @@ // Progressive enhancement of the generated pages: table captions, filters -// and keyboard scrolling, the hash lookup on files/, moved cards, a hash -// typed in the search, glossary terms on a tap, and the 404 page's hints. +// and keyboard scrolling, the hash lookup on files/ (a pasted hash or a file +// hashed on this device), moved cards, a hash typed in the search, glossary +// terms on a tap, the OS tabs kept in view, and the 404 page's hints. // Which tables stack on a phone is decided when the page is built // (src/plugins/rehype-stack-tables.mjs), so the layout is right on first paint. const HASH_LENGTHS: Record = { 8: 'CRC32', 32: 'MD5', 40: 'SHA1', 64: 'SHA256' }; -const LAYER_LABELS: Record = { - collection: 'In the collection', - 'data-directory': 'In a data directory the packs ship', - platform: 'Declared by a platform', - emulator: 'Read by an emulator', +// One line of api/v1/lookup/. `system_name` is read when a record carries it. +type Item = { + algorithm: string; + layer: string; + name?: string; + value: string; + url: string; + owner?: string; + sha1?: string; + system_name?: string; }; -type Item = { algorithm: string; layer: string; name?: string; value: string; url: string; owner?: string }; const siteRoot = () => new URL(import.meta.env.BASE_URL.replace(/\/?$/, '/'), location.origin); const normalizeHash = (raw: string) => raw.trim().toLowerCase().replace(/^0x/, '').replace(/\s+/g, ''); @@ -75,26 +80,17 @@ const setScrollable = (table: HTMLTableElement, overflows: boolean) => { } }; -const filterTable = (table: HTMLTableElement, index: number, heading: string) => { +// The filter box above a long table is written when the page is built +// (src/plugins/rehype-stack-tables.mjs), so the table never moves down once +// drawn. Here it only filters. +const filterTable = (table: HTMLTableElement) => { + const controls = table.previousElementSibling; + if (!controls?.matches('.rb-table-filter')) return; + const input = controls.querySelector('input'); + const count = controls.querySelector('.rb-table-filter-count'); + if (!input || !count || input.dataset.rbWired) return; + input.dataset.rbWired = 'true'; const rows = [...table.querySelectorAll('tbody tr')]; - if (rows.length < 20) return; - const controls = document.createElement('div'); - controls.className = 'rb-table-filter'; - const id = `rb-table-filter-${index}`; - const label = document.createElement('label'); - label.htmlFor = id; - label.textContent = `Filter ${heading}`; - const input = document.createElement('input'); - input.id = id; - input.type = 'search'; - input.autocomplete = 'off'; - input.placeholder = 'Name, platform, system, hash...'; - const count = document.createElement('span'); - count.className = 'rb-table-filter-count'; - count.setAttribute('aria-live', 'polite'); - count.textContent = `${rows.length} rows`; - controls.append(label, input, count); - table.before(controls); // Each row's text is read on the first keystroke, not on every page load. let searchable: { row: HTMLTableRowElement; text: string }[] | null = null; input.addEventListener('input', () => { @@ -105,14 +101,31 @@ const filterTable = (table: HTMLTableElement, index: number, heading: string) => entry.row.hidden = !!query && !entry.text.includes(query); if (!entry.row.hidden) visible += 1; } - count.textContent = `${visible} of ${rows.length} rows`; + count.textContent = query + ? `${visible.toLocaleString('en')} of ${rows.length.toLocaleString('en')} rows` + : `${rows.length.toLocaleString('en')} rows`; }); }; +// A stacked table of many rows leaves its column names out of the HTML +// (rehype-stack-tables.mjs): each cell gets its own here, as the table nears +// the screen. The name sits in the left room every stacked cell keeps, so +// writing it moves nothing. +const labelCells = (table: HTMLTableElement) => { + if (table.dataset.rbStack !== 'true') return; + const labels = [...table.querySelectorAll('thead th')].map((cell) => cell.textContent?.trim() ?? ''); + for (const row of table.tBodies[0]?.rows ?? []) { + [...row.cells].forEach((cell, column) => { + if (labels[column] && !cell.dataset.label) cell.dataset.label = labels[column]; + }); + } +}; + const enhanceTables = () => { const tables = [...document.querySelectorAll('.sl-markdown-content table')]; const enhance = (table: HTMLTableElement) => { for (const cell of table.querySelectorAll('thead th')) if (!cell.hasAttribute('scope')) cell.setAttribute('scope', 'col'); + labelCells(table); const heading = tableName(table); if (!table.querySelector('caption') && !table.closest('.rb-meta-card')) { const caption = document.createElement('caption'); @@ -120,7 +133,7 @@ const enhanceTables = () => { caption.textContent = heading; table.prepend(caption); } - filterTable(table, tables.indexOf(table), heading); + filterTable(table); }; // A table is set up and measured when it nears the screen. Doing all of // them at load relaid out every row of a 3,800-row page after its first @@ -159,16 +172,100 @@ const enhanceTables = () => { }); }; +// Hashing a file on this device: SHA1 and SHA256 by Web Crypto, MD5 and +// CRC32 here, since Web Crypto has neither. The bytes never leave the page. + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n += 1) { + let c = n; + for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + table[n] = c >>> 0; + } + return table; +})(); + +const crc32 = (bytes: Uint8Array): string => { + let crc = 0xffffffff; + for (let i = 0; i < bytes.length; i += 1) crc = CRC_TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8); + return ((crc ^ 0xffffffff) >>> 0).toString(16).padStart(8, '0'); +}; + +const MD5_SHIFTS = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]; +const MD5_TABLE = Array.from({ length: 64 }, (_, i) => Math.floor(Math.abs(Math.sin(i + 1)) * 2 ** 32) >>> 0); + +const hexBytes = (bytes: Uint8Array) => [...bytes].map((byte) => byte.toString(16).padStart(2, '0')).join(''); + +// RFC 1321, block by block over the file's own buffer: no padded copy of a +// 500 MB file. +const md5 = (bytes: Uint8Array): string => { + const state = new Int32Array([0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476]); + const words = new Int32Array(16); + const block = (view: DataView, offset: number) => { + for (let i = 0; i < 16; i += 1) words[i] = view.getInt32(offset + i * 4, true); + let [a, b, c, d] = state; + for (let i = 0; i < 64; i += 1) { + const round = i >> 4; + const f = round === 0 ? (b & c) | (~b & d) : round === 1 ? (d & b) | (~d & c) : round === 2 ? b ^ c ^ d : c ^ (b | ~d); + const g = round === 0 ? i : round === 1 ? (5 * i + 1) % 16 : round === 2 ? (3 * i + 5) % 16 : (7 * i) % 16; + const shift = MD5_SHIFTS[round * 4 + (i % 4)]; + const sum = (a + f + MD5_TABLE[i] + words[g]) | 0; + a = d; + d = c; + c = b; + b = (b + ((sum << shift) | (sum >>> (32 - shift)))) | 0; + } + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + }; + const whole = bytes.length - (bytes.length % 64); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + for (let offset = 0; offset < whole; offset += 64) block(view, offset); + const tail = new Uint8Array(bytes.length - whole < 56 ? 64 : 128); + tail.set(bytes.subarray(whole)); + tail[bytes.length - whole] = 0x80; + const tailView = new DataView(tail.buffer); + const bits = bytes.length * 8; + tailView.setUint32(tail.length - 8, bits >>> 0, true); + tailView.setUint32(tail.length - 4, Math.floor(bits / 2 ** 32), true); + for (let offset = 0; offset < tail.length; offset += 64) block(tailView, offset); + return hexBytes(new Uint8Array(state.buffer)); +}; + +type FileHashes = { sha1: string; md5: string; crc32: string; sha256: string }; + +const hashFile = async (file: Blob): Promise => { + const buffer = await file.arrayBuffer(); + const bytes = new Uint8Array(buffer); + const [sha1, sha256] = await Promise.all( + ['SHA-1', 'SHA-256'].map(async (name) => hexBytes(new Uint8Array(await crypto.subtle.digest(name, buffer)))), + ); + return { sha1, md5: md5(bytes), crc32: crc32(bytes), sha256 }; +}; + // files/?hash= reads one shard of the lookup, named by the first two hex -// characters of the value. +// characters of the value. ?md5=, ?crc32= and ?sha256= are read the same way. + +const HASH_PARAMS = ['hash', 'md5', 'crc32', 'sha256']; const lookupShard = (value: string): Promise => fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot())) .then((response) => (response.ok ? response.json() : { items: [] })) .then((shard) => shard.items || []); +const lookupValues = async (values: string[]): Promise => { + const prefixes = [...new Set(values.map((value) => value.slice(0, 2)))]; + const shards = await Promise.all(prefixes.map(lookupShard)); + const wanted = new Set(values); + return shards.flat().filter((item) => wanted.has(item.value)); +}; + type Held = { name: string; path: string; sha1: string; size: number }; type Declared = { name?: string; destination?: string; required?: boolean; [hash: string]: unknown }; +type ProfileFile = { name?: string; region?: string | string[]; category?: string; [field: string]: unknown }; +type Reader = { name: string; files: ProfileFile[] }; const json = (path: string) => fetch(new URL(path, siteRoot())).then((response) => (response.ok ? response.json() : Promise.reject(response.status))); @@ -204,15 +301,33 @@ const strong = (text: string) => { return element; }; +const count = (n: number, one: string, many: string) => `${n.toLocaleString('en')} ${n === 1 ? one : many}`; + +const listed = (words: string[]) => + words.length < 3 ? words.join(' and ') : `${words.slice(0, -1).join(', ')} and ${words.at(-1)}`; + +// A region: value in words, as the system pages write it. +const REGION_WORDS: Record = { uk: 'UK', 'asia-ntsc': 'Asia (NTSC)', 'asia-pal': 'Asia (PAL)' }; +const regionWord = (region: string) => + REGION_WORDS[region] ?? region.replace(/-/g, ' ').replace(/\b\w/g, (letter) => letter.toUpperCase()); +const CATEGORY_WORDS: Record = { bios: 'BIOS', game_data: 'game data', bios_zip: 'BIOS ZIP' }; + // The card a collection match points at, read from its manufacturer's file -// of api/v1/systems/: the path in the collection and the size. -const heldFile = async (item: Item & { sha1?: string }): Promise => { +// of api/v1/systems/: the path in the collection and the size. A console's +// own page (systems/sony-playstation/) is part of its manufacturer's file +// (systems/sony.json), found by dropping words from the end of the slug. +const heldFile = async (item: Item): Promise => { const slug = /^systems\/([^/]+)\//.exec(item.url)?.[1]; if (!slug || !item.sha1) return undefined; - const data = await json(`api/v1/systems/${slug}.json`); - for (const files of Object.values(data.items?.[0]?.consoles ?? {})) { - const found = files.find((file) => file.sha1 === item.sha1); - if (found) return found; + const words = slug.split('-'); + for (let size = words.length; size > 0; size -= 1) { + const data = await json(`api/v1/systems/${words.slice(0, size).join('-')}.json`).catch(() => undefined); + if (!data) continue; + for (const files of Object.values(data.items?.[0]?.consoles ?? {})) { + const found = files.find((file) => file.sha1 === item.sha1); + if (found) return found; + } + return undefined; } return undefined; }; @@ -220,15 +335,40 @@ const heldFile = async (item: Item & { sha1?: string }): Promise => { +const declaredEntry = async (item: Item): Promise => { const system = decodeURIComponent(new URL(item.url, siteRoot()).hash.slice(1)); const data = await json(`api/v1/platforms/${item.owner}.json`); const files: Declared[] = data.items?.[0]?.contract?.systems?.[system]?.files ?? []; const holds = (entry: Declared) => - String(entry[item.algorithm] ?? '').toLowerCase().split(',').map((hash) => hash.trim()).includes(value); + String(entry[item.algorithm] ?? '').toLowerCase().split(',').map((hash) => hash.trim()).includes(item.value); return files.find((entry) => holds(entry) && entry.name === item.name) ?? files.find(holds); }; +// An emulator by the name its page carries, with the entries of its profile. +const readerProfile = async (owner: string): Promise => { + const data = await json(`api/v1/emulators/${owner}.json`).catch(() => undefined); + const profile = data?.items?.[0]?.profile; + if (!profile) return undefined; + return { name: String(profile.emulator || profile.display_name || owner), files: profile.files ?? [] }; +}; + +// The entry of a profile that reads the value: by name and hash, else by hash. +const readerEntry = (reader: Reader | undefined, item: Item): ProfileFile | undefined => { + const holds = (entry: ProfileFile) => + String(entry[item.algorithm] ?? '').toLowerCase().split(',').map((hash) => hash.trim()).includes(item.value); + return reader?.files.find((entry) => holds(entry) && entry.name === item.name) ?? reader?.files.find(holds); +}; + +// One value every reader that says something agrees on, else nothing. +const agreed = (values: (string | undefined)[]): string | undefined => { + const said = new Set(values.filter((value): value is string => !!value)); + return said.size === 1 ? [...said][0] : undefined; +}; + +// A name a setup writes in place of a file name: EmuDeck lists a hash under +// "system:md5", and any file that has it passes. +const anyName = (name: string) => !name || name.includes(':'); + const setupTable = (rows: { item: Item; entry?: Declared; setup: string }[]) => { const table = document.createElement('table'); const head = table.createTHead().insertRow(); @@ -242,12 +382,10 @@ const setupTable = (rows: { item: Item; entry?: Declared; setup: string }[]) => for (const { item, entry, setup } of rows) { const row = body.insertRow(); const name = entry?.name ?? item.name ?? ''; - // EmuDeck lists a hash with no file name: any file that has it passes. - const named = name && !name.includes(':'); const destination = entry?.destination; const cells: (Node | string)[] = [ link(item.url, setup), - named ? code(name) : 'Any name', + anyName(name) ? 'Any name' : code(name), destination ? code(destination) : '-', item.algorithm.toUpperCase(), entry?.required === undefined ? '-' : entry.required ? 'Yes' : 'No', @@ -262,30 +400,83 @@ const setupTable = (rows: { item: Item; entry?: Declared; setup: string }[]) => return table; }; -// The answer first: which file this is and where the collection holds it, -// then the setups that list it (where it goes, whether it is required), -// then the emulators that read it. -const renderMatches = async (status: HTMLElement, result: HTMLElement, value: string, items: Item[]) => { +// One declaration per setup, name and section: a value found by two of its +// hashes (a file hashed here) is still one line of the setup's list. +const distinct = (items: Item[], key: (item: Item) => string) => { + const seen = new Map(); + for (const item of items) if (!seen.has(key(item))) seen.set(key(item), item); + return [...seen.values()]; +}; + +// The answer first: which file this is, for which system and region, and +// how many setups list it under which names. Then where the collection +// holds it, the setups that list it (where it goes, whether it is +// required), and the emulators that read it, by name. +const renderMatches = async (status: HTMLElement, result: HTMLElement, items: Item[]) => { const byLayer = (layer: string) => items.filter((item) => item.layer === layer); - const held = byLayer('collection'); - const platforms = byLayer('platform'); - const [files, entries, names] = await Promise.all([ + const held = distinct(byLayer('collection'), (item) => item.sha1 ?? item.url); + const platforms = distinct(byLayer('platform'), (item) => `${item.owner}|${item.name}|${item.url}`); + const emulators = distinct(byLayer('emulator'), (item) => `${item.owner}|${item.name}|${item.url}`); + const owners = [...new Set(emulators.map((item) => item.owner ?? ''))]; + const [files, entries, names, readers] = await Promise.all([ Promise.all(held.map((item) => heldFile(item).catch(() => undefined))), - Promise.all(platforms.map((item) => declaredEntry(item, value).catch(() => undefined))), + Promise.all(platforms.map((item) => declaredEntry(item).catch(() => undefined))), json('data/picker/index.json').catch(() => []) as Promise<{ platform: string; name: string }[]>, + Promise.all(owners.map((owner) => readerProfile(owner))), ]); const setupName = new Map(names.map((entry) => [entry.platform, entry.name])); - status.textContent = `${items.length.toLocaleString('en')} ${items.length === 1 ? 'match' : 'matches'} for ${HASH_LENGTHS[value.length]} ${value}.`; + const reader = new Map(owners.map((owner, index) => [owner, readers[index]])); + const readEntries = emulators.map((item) => readerEntry(reader.get(item.owner ?? ''), item)); + const setups = new Set(platforms.map((item) => item.owner)); + + const counted = [ + held.length ? count(held.length, 'file in the collection', 'files in the collection') : 'not in the collection', + ...(platforms.length ? [`${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`] : []), + ...(owners.length ? [count(owners.length, 'emulator that reads it', 'emulators that read it')] : []), + ]; + status.textContent = `Found: ${counted.join(', ')}.`; + const blocks: Node[] = []; + const category = agreed(readEntries.map((entry) => entry && CATEGORY_WORDS[String(entry.category ?? 'bios')])); + const regions = agreed(readEntries.map((entry) => { + const raw = entry?.region; + const list = Array.isArray(raw) ? raw : raw ? [String(raw)] : []; + return list.length ? listed(list.map(regionWord)) : undefined; + })); held.forEach((item, index) => { const file = files[index]; - const name = file?.name ?? item.name ?? value; + const name = file?.name ?? item.name ?? item.value; + const system = item.system_name; + const what: (Node | string)[] = system + ? [', ', link(item.url, system), category ? ` ${category}` : ' file'] + : category ? [`, a ${category} file`] : []; + const others = new Map>(); + for (const declared of platforms) { + const other = declared.name ?? ''; + if (anyName(other) || other === name) continue; + if (!others.has(other)) others.set(other, new Set()); + others.get(other)!.add(declared.owner ?? ''); + } + const renamed = new Set([...others.values()].flatMap((set) => [...set])); + const otherNames = [...others.keys()].sort(); + const named: (Node | string)[] = []; + otherNames.forEach((other, position) => { + if (position) named.push(position === otherNames.length - 1 ? ' and ' : ', '); + named.push(code(other)); + }); + const listing: (Node | string)[] = setups.size + ? [ + ` ${count(setups.size, 'setup lists', 'setups list')} it`, + ...(renamed.size + ? [`, ${renamed.size.toLocaleString('en')} of them under the ${otherNames.length === 1 ? 'name' : 'names'} `, ...named] + : []), + '.', + ] + : []; blocks.push( - file - ? paragraph('This is ', code(name), ', held in the collection at ', code(file.path), '.') - : paragraph('This is ', code(name), ', held in the collection.'), + paragraph('This is ', code(name), ...what, ...(regions ? [` for ${regions}`] : []), '.', ...listing), paragraph( - ...(file ? ['SHA1 ', code(file.sha1), `, ${formatSize(file.size)}. `] : []), + ...(file ? ['Held in the collection at ', code(file.path), '. SHA1 ', code(file.sha1), `, ${formatSize(file.size)}. `] : []), link(item.url, 'Open its card'), '.', ), @@ -300,12 +491,11 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, value: st row.append(link(item.url, code(item.name || item.value))); list.append(row); } - blocks.push(paragraph(strong(LAYER_LABELS['data-directory'])), list); + blocks.push(paragraph(strong('In a data directory the packs ship')), list); } if (platforms.length) { - const owners = new Set(platforms.map((item) => item.owner)); blocks.push( - paragraph(strong(`Setups that list it (${owners.size})`)), + paragraph(strong('Setups that list it: '), `${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`), setupTable(platforms.map((item, index) => ({ item, entry: entries[index], @@ -313,20 +503,96 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, value: st }))), ); } - const readers = byLayer('emulator'); - if (readers.length) { + if (owners.length) { const list = document.createElement('ul'); - for (const item of readers) { + emulators.forEach((item) => { const row = document.createElement('li'); - row.append(link(item.url, item.owner ?? item.name ?? '')); - if (item.name && item.name !== held[0]?.name) row.append(' as ', code(item.name)); + row.append(link(item.url, reader.get(item.owner ?? '')?.name ?? item.owner ?? item.name ?? '')); + if (item.name && item.name !== (files[0]?.name ?? held[0]?.name)) row.append(' as ', code(item.name)); list.append(row); - } - blocks.push(paragraph(strong(`${LAYER_LABELS.emulator} (${readers.length})`)), list); + }); + blocks.push(paragraph(strong(`Emulators that read it (${owners.length.toLocaleString('en')})`)), list); } result.replaceChildren(...blocks); }; +const WANTED = 'provenance/#wanted-catalogued-dumps-this-project-does-not-have'; + +// Nothing the site knows has the value: where a dump of one's own can go. +const renderNoMatch = (status: HTMLElement, result: HTMLElement) => { + status.textContent = 'No file, setup or emulator the site knows has this hash.'; + result.replaceChildren( + paragraph('If you dumped it yourself, ', link('contributing/', 'add it'), '.'), + paragraph('The ', link(WANTED, 'wanted list'), ' names the catalogued dumps the collection still lacks.'), + ); +}; + +const lookUp = (status: HTMLElement, result: HTMLElement, values: string[]) => { + status.textContent = 'Looking it up...'; + result.replaceChildren(); + return lookupValues(values) + .then((matches) => (matches.length ? renderMatches(status, result, matches) : renderNoMatch(status, result))) + .catch(() => { + status.textContent = 'The file index could not be read.'; + }); +}; + +// A file chosen on files/ is hashed here, its four hashes shown, and all +// four looked up: a setup may list only the MD5. +const MAX_HASHED = 512 * 1024 ** 2; + +const fileHasher = (status: HTMLElement, result: HTMLElement) => { + const form = document.querySelector('form.rb-file-lookup'); + if (!form) return; + const box = document.createElement('div'); + box.className = 'rb-file-pick'; + const label = document.createElement('label'); + label.htmlFor = 'rb-file'; + label.textContent = 'Or choose the file'; + const input = document.createElement('input'); + input.type = 'file'; + input.id = 'rb-file'; + const note = paragraph('It is hashed on this device. Nothing is uploaded.'); + note.className = 'rb-file-pick-note'; + const hashes = document.createElement('p'); + hashes.className = 'rb-file-pick-hashes'; + hashes.hidden = true; + box.append(label, input, note, hashes); + form.after(box); + input.addEventListener('change', async () => { + const file = input.files?.[0]; + if (!file) return; + hashes.hidden = true; + if (file.size > MAX_HASHED) { + status.textContent = `This page hashes files up to ${formatSize(MAX_HASHED)}. ${file.name} is ${formatSize(file.size)}.`; + result.replaceChildren(); + return; + } + status.textContent = `Hashing ${file.name}...`; + result.replaceChildren(); + let found: FileHashes; + try { + found = await hashFile(file); + } catch { + status.textContent = `${file.name} could not be read.`; + return; + } + hashes.replaceChildren( + strong(file.name), ` ${formatSize(file.size)}`, + ...(['sha1', 'md5', 'crc32', 'sha256'] as const).flatMap((algorithm) => [ + document.createElement('br'), `${algorithm.toUpperCase()} `, code(found[algorithm]), + ]), + ); + hashes.hidden = false; + const field = document.getElementById('rb-hash') as HTMLInputElement | null; + if (field) field.value = found.sha1; + const url = new URL(location.href); + url.search = `?hash=${found.sha1}`; + history.replaceState(null, '', url); + await lookUp(status, result, [found.sha1, found.md5, found.crc32, found.sha256]); + }); +}; + // files/?sha1= is a file's content address and opens its card // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and // lists every page that names the value. @@ -336,9 +602,11 @@ const resolveFileAddress = () => { const result = document.createElement('div'); result.id = 'rb-file-lookup-result'; status.after(result); + fileHasher(status, result); const params = new URLSearchParams(location.search); const permalink = params.has('sha1'); - const value = normalizeHash(params.get('sha1') || params.get('hash') || ''); + const raw = params.get('sha1') || HASH_PARAMS.map((key) => params.get(key)).find(Boolean) || ''; + const value = normalizeHash(raw); if (!value) return; const input = document.getElementById('rb-hash') as HTMLInputElement | null; if (input) input.value = value; @@ -347,18 +615,22 @@ const resolveFileAddress = () => { status.textContent = 'A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters.'; return; } + if (!prefix && !permalink) { + lookUp(status, result, [value]); + return; + } status.textContent = 'Looking it up...'; lookupShard(value) .then((items) => { const matches = items.filter((item) => prefix ? item.algorithm === 'sha1' && item.value.startsWith(value) : item.value === value); const cards = matches.filter((item) => item.layer === 'collection' || item.layer === 'data-directory'); - if (permalink && new Set(cards.map((item) => item.url)).size === 1) { + if (new Set(cards.map((item) => item.url)).size === 1) { location.replace(new URL(cards[0].url, siteRoot())); } else if (matches.length) { - return renderMatches(status, result, prefix ? matches[0].value : value, prefix ? cards : matches); + return renderMatches(status, result, prefix ? cards : matches); } else { - status.textContent = 'No file the site knows has this hash.'; + renderNoMatch(status, result); } }) .catch(() => { @@ -383,32 +655,52 @@ const followMovedCard = () => { }; // The site search holds no hashes. A hash typed in it gets "Look up this -// hash" as its first result, and Enter opens the lookup. -const hashLookupRow = (drawer: Element, value: string) => { +// hash" as its first result, and Enter opens the lookup. The pages Pagefind +// finds for the same characters (a file name that holds them, a word that +// starts like them) are no answer: they stay folded under a button that +// says what they are. +const hashLookupRow = (drawer: HTMLElement, value: string) => { let row = drawer.querySelector(':scope > .rb-search-hash'); if (!isHash(value)) { row?.remove(); + delete drawer.dataset.rbHash; + delete drawer.dataset.rbPagesShown; return; } + drawer.dataset.rbHash = ''; + const target = new URL(`files/?hash=${value}`, siteRoot()).href; + if (row?.querySelector('a')?.href === target) return; if (!row) { - row = document.createElement('p'); + row = document.createElement('div'); row.className = 'rb-search-hash'; drawer.prepend(row); } - const target = new URL(`files/?hash=${value}`, siteRoot()).href; - if (row.querySelector('a')?.href === target) return; + delete drawer.dataset.rbPagesShown; const anchor = document.createElement('a'); anchor.href = target; anchor.append('Look up this hash'); - row.replaceChildren(anchor, ` ${HASH_LENGTHS[value.length]} `, code(value)); + const toggle = document.createElement('button'); + toggle.type = 'button'; + toggle.className = 'rb-search-pages'; + toggle.setAttribute('aria-expanded', 'false'); + toggle.textContent = 'Pages that mention these characters'; + toggle.addEventListener('click', () => { + const open = toggle.getAttribute('aria-expanded') !== 'true'; + toggle.setAttribute('aria-expanded', String(open)); + if (open) drawer.dataset.rbPagesShown = ''; + else delete drawer.dataset.rbPagesShown; + }); + row.replaceChildren(paragraph(anchor, ` ${HASH_LENGTHS[value.length]} `, code(value)), toggle); }; // Pagefind lists a page's title twice: once for the page and once for the // anchor of its h1 (#_top). The second title is hidden, its excerpt kept. -// The `files` field of a system page weighs its file names in the ranking -// and is never shown: a result is not a list of 120 names. Pagefind reuses +// The `files`, `aliases` and `devices` fields weigh in the ranking and are +// never shown: a result is not a list of 120 names. Pagefind reuses // its nodes from one query to the next, so both are worked out again on // every change. +const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices']); + const tidyResults = (root: Element) => { for (const title of root.querySelectorAll('.pagefind-ui__result-nested > .pagefind-ui__result-title')) { title.hidden = !!title.querySelector('a')?.getAttribute('href')?.endsWith('#_top'); @@ -416,7 +708,7 @@ const tidyResults = (root: Element) => { for (const tags of root.querySelectorAll('.pagefind-ui__result-tags')) { let shown = 0; for (const tag of tags.querySelectorAll('.pagefind-ui__result-tag')) { - tag.hidden = tag.dataset.pagefindUiMeta === 'files'; + tag.hidden = UNSHOWN_FIELDS.has(tag.dataset.pagefindUiMeta ?? ''); if (!tag.hidden) shown += 1; } tags.hidden = shown === 0; @@ -429,7 +721,7 @@ const hashSearch = () => { const refresh = () => { tidyResults(root); const input = root.querySelector('input'); - const drawer = root.querySelector('.pagefind-ui__drawer'); + const drawer = root.querySelector('.pagefind-ui__drawer'); if (input && drawer) hashLookupRow(drawer, normalizeHash(input.value)); }; new MutationObserver(refresh).observe(root, { @@ -457,6 +749,31 @@ const hashSearch = () => { ); }; +// The OS tabs scroll sideways on a phone instead of wrapping to a second +// row (retrobios.css): the chosen one is kept in view, on load and on a +// change, by scrolling the row and never the page. +const keepTabInView = (tab: Element) => { + const list = tab.closest('[role="tablist"]'); + if (!list) return; + const row = list.getBoundingClientRect(); + const box = tab.getBoundingClientRect(); + if (box.left < row.left) list.scrollLeft -= row.left - box.left; + else if (box.right > row.right) list.scrollLeft += box.right - row.right; +}; + +const osTabs = () => { + const selected = () => document.querySelectorAll('starlight-tabs [role="tab"][aria-selected="true"]').forEach(keepTabInView); + if (!document.querySelector('starlight-tabs')) return; + const later = () => requestAnimationFrame(selected); + for (const type of ['click', 'keyup']) { + document.addEventListener(type, (event) => { + if ((event.target as Element | null)?.closest?.('starlight-tabs [role="tab"]')) later(); + }); + } + if (document.readyState === 'complete') later(); + else addEventListener('load', later, { once: true }); +}; + // A glossary term shows its definition on hover; a tap or Enter shows it // on a phone or to a keyboard, and Escape or a tap elsewhere closes it. const glossaryTerms = () => { @@ -505,4 +822,5 @@ followMovedCard(); enhanceTables(); hashSearch(); glossaryTerms(); +osTabs(); notFound(); diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index c3f5be83..251eb6a9 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` | | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_css`, `test_site_r3_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture