feat: find a file by any hash, out of the search

Hashes, link lists, install commands, member tables and long reference
tables leave the site search: the index drops from 21.0 MB (4.88 MB
gzip) to 6.8 MB (1.33 MB gzip). Names and prose stay searchable.

files/?hash= takes a SHA1, MD5, CRC32 or SHA256 and reads one shard of
api/v1/lookup/ (about 22 KB gzip, was 675 KB) listing the collection
card, the platforms that declare the value and the profiles that check
it. files/?sha1= still opens the card, and a sha1- anchor missing from
a page is looked up there, so a card can move between pages.

Excluded blocks are section elements: the search plugin compares
elements by tag, so a div ended at the first nested div.
This commit is contained in:
Abdessamad Derraz committed 2026-10-22 12:46:40 +02:00
1 parent 42a0498d20
commit 33041b6bd1
9 files changed
+481 -67

No files matched your search

+36 -13
View File
@@ -339,9 +339,10 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
class TheSearchSkipsLinkLists(unittest.TestCase):
"""The search index was 24.7 MB, read and indexed before the first
result. Lists of platforms, copies and source citations on every card
made 6.6 MB of it, names already indexed on their own pages."""
made 6.6 MB of it, names already indexed on their own pages, and the
hashes 4 MB more: files/?hash= finds those."""
def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self):
def test_a_system_card_keeps_its_name_and_skips_its_body(self):
import generate_site
from cross_reference import FileConsumers
@@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase):
consumers.emulators = {sha1: {"duckstation"}}
entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3}
text = "\n".join(generate_site._system_file_card(entry, consumers))
searched, skipped = text.split('<div class="rb-search-skip" data-search-exclude markdown>')
self.assertIn("- MD5:", searched)
self.assertIn("- Emulators:", searched)
self.assertIn("- Platforms:", skipped)
self.assertIn("- Also installed by:", skipped)
self.assertEqual(text.count("</div>"), 2)
searched, skipped = text.split("<section data-search-exclude markdown>")
self.assertIn("`a.bin`", searched)
self.assertNotIn("- MD5:", searched)
for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"):
self.assertIn(line, skipped)
def test_a_game_data_card_skips_its_name_too(self):
import generate_site
from cross_reference import FileConsumers
entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3}
lines = generate_site._system_file_card(
entry, FileConsumers({"files": {}}), searchable=False)
self.assertTrue(lines[2].endswith("data-search-exclude }"))
def test_runs_alternate_in_card_order(self):
import generate_site
lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
"- Destination: `d`", "- Contents (1 members):", " - `m`"]
lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
"- Destination: `d`", "- Contents (1 members):", " - `m`",
"- Note"]
out = [line for line in generate_site._search_skipped_apart(lines) if line]
self.assertEqual(out, [
"- SHA1: x",
"- Path: `p`",
'<div class="rb-search-skip" data-search-exclude markdown>',
"- Source: `a.c:1`", "- Platforms: p", "</div>",
"- Destination: `d`", "- Contents (1 members):", " - `m`",
"- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "</div>",
"- Destination: `d`",
'<div class="rb-search-skip" data-search-exclude markdown>',
"- Contents (1 members):", " - `m`", "</div>",
"- Note",
])
def test_long_tables_leave_the_search(self):
import generate_site
short = "| a |\n|---|\n| 1 |"
long = " | a |\n |---|\n" + "\n".join(f" | {'x' * 50} |" for _ in range(80))
text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n")
self.assertEqual(text.count("data-search-exclude"), 1)
self.assertIn(" <section data-search-exclude markdown>\n\n | a |", text)
self.assertIn(short, text)
if __name__ == "__main__":
unittest.main()