From 2f721bc640d3ea7425c4dffd4d0e697dea379f17 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:16:44 +0100
Subject: [PATCH] fix: list unsourceable files as not collected yet
Every card read "Not collectable" and the gaps page listed 259 entries as
acknowledged gaps verified from source code. unsourceable: only keeps a
same-named file from being served in the entry's place, and each entry
stays an acquisition target (wiki/profiling.md). The section now names
the catalog or romset that lists each file, from the provenance report
built with the profiles' needs: XM7's KANJI2.ROM is in No-Intro and in
MAME's fm7740sx.zip. The old anchor still leads to the section.
---
scripts/generate_site.py | 74 +++++++++++++++++++++++----------
tests/test_audit_regressions.py | 28 ++++++++++++-
2 files changed, 79 insertions(+), 23 deletions(-)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 62f400543..0eece3cfc 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -86,7 +86,7 @@ from generate_readme import (
extract_targets,
release_totals,
)
-from provenance_report import build_report, distinct_missing
+from provenance_report import build_report, distinct_missing, profile_needs
import sitemarkup as markup
import siteentries
import siteinputs
@@ -8433,12 +8433,7 @@ def _render_emulator_file(
if fnote:
lines.append(f"- {data_root_words(_escape_tags(link_prose_citations(profile, str(fnote))), data_href)}")
if not in_repo and f.get("unsourceable"):
- # The badge alone sent readers after a dump that cannot exist, and the
- # reason was only on the gaps page, which the card did not link.
- lines.append(
- f"- Not collectable: {_escape_tags(str(f['unsourceable']))} "
- "([acknowledged gaps](../gaps.md#acknowledged-gaps))"
- )
+ lines.append(f"- {_unsourceable_line(f)}")
lines.extend(_contents_lines(contents))
lines[body_start:] = _search_skipped_apart(lines[body_start:])
lines.append("")
@@ -8550,12 +8545,7 @@ def _emulator_card_folds(
open_lines.append(data_root_words(
_escape_tags(link_prose_citations(profile, str(fnote))), data_href))
if not in_repo and f.get("unsourceable"):
- # The badge alone sent readers after a dump that cannot exist, and the
- # reason was only on the gaps page, which the card did not link.
- open_lines.append(
- f"Not collectable: {_escape_tags(str(f['unsourceable']))} "
- "([acknowledged gaps](../gaps.md#acknowledged-gaps))"
- )
+ open_lines.append(_unsourceable_line(f))
lines = [f"- {d}" for d in open_lines]
shut_fold = [f"- {d}" for d in hashes + [m for m in more if skipped(m)]]
read_fold = [f"- {m}" for m in more if not skipped(m)]
@@ -10459,6 +10449,33 @@ def _hash_cell(hashes: dict[str, list[str]]) -> str:
)
+def _unsourceable_line(f: dict) -> str:
+ """Why the collection holds no copy of a file a profile marks
+ `unsourceable:`, linked to the list of them. The flag only keeps a
+ same-named file from being served in its place: the file is still
+ wanted, and "Not collectable" sent readers away from KANJI2.ROM, which
+ MAME's fm7740sx.zip carries."""
+ return (f"Not collected yet: {_escape_tags(str(f['unsourceable']))} "
+ "([files not collected yet](../gaps.md#not-collected-yet))")
+
+
+def _catalogued_needs(report: dict | None) -> dict[tuple[str, str], list[str]]:
+ """(profile, file name casefolded) -> where a dump catalog or an
+ imported romset lists that file, from a provenance report built with
+ the profiles' needs."""
+ out: dict[tuple[str, str], list[str]] = {}
+ for source, data in sorted((report or {}).items()):
+ label = _PROVENANCE_LABELS.get(source, source)
+ for entry in data.get("missing", []):
+ where = [f"{label} `{_escape_tags(str(entry.get('name', '')))}`"]
+ where += [f"{_escape_tags(str(hit['source']))} `{_escape_tags(str(hit['archive']))}`"
+ for hit in entry.get("romsets", [])]
+ for need in entry.get("needed_by", []):
+ held = out.setdefault((need["emulator"], str(need["name"]).casefold()), [])
+ held += [w for w in where if w not in held]
+ return out
+
+
def _reason_cell(text: str) -> str:
"""A reason in a gaps table, in the class that keeps its column readable."""
return f'{text}' if text else "-"
@@ -10753,9 +10770,14 @@ def generate_gap_analysis(
registry: dict | None = None,
gap_report: dict | None = None,
moved: dict[str, str] | None = None,
+ catalogued: dict | None = None,
) -> str:
"""Generate a unified gap analysis page.
+ ``catalogued`` is a provenance report built with the profiles' needs:
+ a file the collection does not hold yet names the catalog or romset
+ that lists it.
+
With ``moved``, the list of files to source goes to pages of its own,
keyed by stem: on the page it made 662 KB of HTML.
@@ -11186,7 +11208,7 @@ def generate_gap_analysis(
"",
])
- # ---- Section 4: Acknowledged gaps (unsourceable files) ----
+ # ---- Section 4: files the collection does not hold yet (unsourceable) ----
all_unsourceable: list[dict] = []
for emu_name, data in sorted(report_all.items()):
@@ -11200,19 +11222,26 @@ def generate_gap_analysis(
})
if all_unsourceable:
+ listed = _catalogued_needs(catalogued)
+ # The old heading's address still leads here.
lines.extend([
- "## Acknowledged gaps",
+ '',
"",
- f"{len(all_unsourceable)} files documented as unsourceable "
- "(verified from source code).",
+ "## Not collected yet",
"",
- "| File | Emulator | Reason | Source ref |",
- "|------|----------|--------|-----------|",
+ f"{len(all_unsourceable)} files the collection does not hold. A file "
+ "with the same name is never served in their place. Each is still "
+ "wanted, and the last column names a catalog or romset that lists it.",
+ "",
+ "| File | Emulator | Reason | Source ref | Listed in |",
+ "|------|----------|--------|-----------|-----------|",
])
for u in sorted(all_unsourceable, key=lambda x: x["name"]):
+ where = listed.get((u["emu_key"], str(u["name"]).casefold()), [])
lines.append(
f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} "
- f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |"
+ f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} "
+ f"| {', '.join(where) or '-'} |"
)
lines.append("")
@@ -12563,12 +12592,16 @@ def _generate(args: argparse.Namespace) -> None:
sitestarlight.MOVED_PAGES.setdefault(
f"cross-reference-{pname}.md", f"platforms/{pname}.md")
+ provenance_report = build_report(
+ db, load_provenance_snapshots(), needs=profile_needs(profiles, db))
+
print("Generating gap analysis page...")
gap_pages: dict[str, str] = {}
write_if_changed(
str(docs / "gaps.md"),
tables_out_of_search(generate_gap_analysis(
profiles, coverages, db, suppl_names, registry, gap_report, gap_pages,
+ catalogued=provenance_report,
)) + reference_next(),
)
for stem, page in gap_pages.items():
@@ -12584,7 +12617,6 @@ def _generate(args: argparse.Namespace) -> None:
)
print("Generating dump provenance page...")
- provenance_report = build_report(db, load_provenance_snapshots())
provenance_pages: dict[str, str] = {}
write_if_changed(
str(docs / "provenance.md"),
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 3df8850fe..91110a2d5 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5143,8 +5143,32 @@ class UnsourceableCardSaysWhy(unittest.TestCase):
text = "\n".join(_render_emulator_file(
entry, {"files": [entry]}, None, [entry], lambda f: False, "file-x",
))
- self.assertIn("Not collectable: written by the firmware editor", text)
- self.assertIn("../gaps.md#acknowledged-gaps", text)
+ self.assertIn("Not collected yet: written by the firmware editor", text)
+ self.assertIn("../gaps.md#not-collected-yet", text)
+ self.assertNotIn("collectable", text)
+
+ def test_the_gaps_page_names_where_a_wanted_file_is_listed(self):
+ """The section read "Acknowledged gaps ... (verified from source
+ code)" while No-Intro lists XM7's KANJI2.ROM and MAME's fm7740sx.zip
+ carries it: the flag only keeps a homonym out, the file is wanted."""
+ from scripts.generate_site import generate_gap_analysis
+
+ unsourced = {"name": "KANJI2.ROM", "reason": "no dump distributed", "source_ref": ""}
+ report = {"xm7": {"emulator": "XM7", "gaps": 0, "gap_missing": 0, "gap_bios": 0,
+ "unsourceable": [unsourced], "gap_details": []}}
+ catalogued = {"no-intro": {"missing": [{
+ "name": "KANJI2.rom",
+ "romsets": [{"source": "mame", "archive": "fm7740sx.zip"}],
+ "needed_by": [{"emulator": "xm7", "name": "KANJI2.ROM", "required": True}],
+ }]}}
+ page = generate_gap_analysis(
+ {"xm7": {"emulator": "XM7", "files": []}}, {}, {"files": {}, "indexes": {}},
+ gap_report=report, catalogued=catalogued)
+ self.assertIn("## Not collected yet", page)
+ self.assertIn('', page)
+ self.assertIn("| No-Intro `KANJI2.rom`, mame `fm7740sx.zip` |", page)
+ self.assertNotIn("Acknowledged", page)
+ self.assertNotIn("verified from source code", page)