kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 07:33:45 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge region and coverage checks

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-10-28 02:08:40 +01:00
2 parents 1ab3fa4fe7 c5a6a0fcc4
commit a1d094c8b3
38 files changed
+5734 -526
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/__init__.py
+5
View File
@@ -16,6 +16,11 @@ import os
from pathlib import Path
os.environ.setdefault("RETROBIOS_OFFLINE", "1")
# A test never waits on, nor blocks, the measurements running on the host.
os.environ.setdefault(
"RETROBIOS_PROBE_LOCK",
str(Path(__file__).resolve().parent.parent / "tmp" / "tests" / "probe.lock"),
)
(Path(__file__).resolve().parent.parent / "tmp" / "tests").mkdir(
parents=True, exist_ok=True
tests/campaign/content.yml
+552 -1
View File
@@ -9,6 +9,8 @@
# region the territory of the disc or cartridge: the BIOS entry the
# boot test varies is the one whose `region:` holds it, or one
# without `region:`
# also_regions the other territories one release was sold in, when the
# catalog names one entry for several (`(USA, Europe)`)
# load the file handed to the core: `cue` (the .cue sheet) or a
# file name inside the folder, by default the single file
# frames frames run per repetition, the same with and without the file
@@ -19,7 +21,18 @@
#
# The checks a game is used for are the boot test (no file against the
# declared file), the files test (every declared file staged, which ones the
# core opens) and the benchmark.
# core opens), the benchmark, and for a system whose BIOS a profile ties to a
# territory, the region check (scripts/region_check.py): one game per region,
# run with only its own region's BIOS, then with only the others.
#
# Systems come in the order of the users a missing BIOS stops: BIOS-required
# consoles first, then the arcade BIOS sets, then everything else with a BIOS
# entry. A system no catalog or romfetch source covers is listed under
# `uncovered`, with the reason and romfetch's error code, so the gap is named
# rather than silent.
#
# A system named twice in the profiles (`sega-megacd` and ares' `mega-cd`)
# repeats its list with a YAML alias: the same games, one download.
systems:
sony-playstation:
@@ -40,3 +53,541 @@ systems:
frames: 3600
frame_hash_every: 10
pinned: true
- name: "Puzzle Bobble 2 (Japan)"
catalog: redump
reason: >-
The Japanese release of the same game as the US and European discs, so
the three region runs show the same title screens and only the BIOS
licence screen differs. Beetle PSX picks scph5500.bin for a Japanese
disc, the region check proves it.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
- name: "Bust-A-Move 2 - Arcade Edition (Europe)"
catalog: redump
reason: >-
The PAL release of the same game: a European disc runs at 50 Hz and
Beetle PSX reads scph5502.bin for it. Third leg of the region check,
beside the Japanese and US discs of the same title.
region: europe
load: cue
frames: 3600
frame_hash_every: 10
sony-playstation-2:
romfetch: ps2
content:
- name: "Rez (USA) (En,Fr,De,Es,It)"
catalog: redump
reason: >-
A 230 MB CD-ROM release, the smallest well-known PS2 disc, that boots
through the PS2 BIOS logo straight to its title screen without input.
The profiles do not tie PS2 BIOS images to a region, so one game is
enough for PCSX2 and LRPS2.
region: north-america
frames: 3600
frame_hash_every: 10
sega-saturn:
romfetch: saturn
content: &saturn
- name: "Puzzle Bobble 2X (Japan) (2M1, 2MB1)"
catalog: redump
reason: >-
The Japanese release of the game the US and European discs carry, so
the three region runs compare the same screens. Beetle Saturn loads
sega_101.bin for a Japanese disc and refuses a disc of another area
code, which the region check shows.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
- name: "Bust-A-Move 2 - Arcade Edition (USA)"
catalog: redump
reason: >-
The US release of the same game: Beetle Saturn loads mpr-17933.bin
for a North American disc. Reaches its title within a minute with no
input.
region: north-america
load: cue
frames: 3600
frame_hash_every: 10
- name: "Bust-A-Move 2 - Arcade Edition (Europe)"
catalog: redump
reason: >-
The European release of the same game, area code E. mpr-17933.bin
also serves PAL consoles in Beetle Saturn, so this disc tells whether
one file covers two territories as the profile says.
region: europe
load: cue
frames: 3600
frame_hash_every: 10
sega-megacd:
romfetch: segacd
content: &megacd
- name: "Sonic the Hedgehog CD (Japan)"
catalog: redump
reason: >-
Sega's own title in all three regions, so the region runs compare one
game. Genesis Plus GX, PicoDrive and the other Mega CD cores choose
bios_CD_J.bin or its aliases for a Japanese console.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
- name: "Sonic CD (USA)"
catalog: redump
reason: >-
The US release of the same game, booted through the Sega CD BIOS
(bios_CD_U.bin, us_scd2_9306.bin in PicoDrive). The BIOS shows its
own animated screen before the disc starts.
region: north-america
load: cue
frames: 3600
frame_hash_every: 10
- name: "Sonic CD (Europe)"
catalog: redump
reason: >-
The PAL release of the same game, for the European Mega CD BIOS
(bios_CD_E.bin, eu_mcd2_9306.bin). Third leg of the region check.
region: europe
load: cue
frames: 3600
frame_hash_every: 10
# ares names the same console mega-cd.
mega-cd:
romfetch: segacd
content: *megacd
sega-32x:
romfetch: 32x
content:
- name: "Knuckles' Chaotix (Japan, USA) (En)"
catalog: no-intro
reason: >-
A 32X-only cartridge: the cores that do not emulate the 32X boot ROMs
in HLE need 32X_G_BIOS.BIN, 32X_M_BIOS.BIN and 32X_S_BIOS.BIN to run
any of it. One release for both territories.
region: north-america
also_regions: [japan]
frames: 3600
frame_hash_every: 10
nec-pc-engine-cd:
romfetch: pcecd
content:
- name: "Gate of Thunder (Japan) (FACT, FADT)"
catalog: redump
reason: >-
A Super CD-ROM2 disc: it needs a System Card 3.0 image (syscard3.pce)
and stops on the card's error screen with an older card or none. The
profiles do not tie the card to a region.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
snk-neogeo-cd:
romfetch: neogeocd
content:
- name: "Metal Slug (Japan) (En,Ja)"
catalog: redump
reason: >-
A Neo Geo CD disc that cannot start without the console's BIOS
(neocd.bin and its CDZ variants). Metal Slug is the title the MVS
entry of the arcade list uses too.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
sega-dreamcast:
romfetch: dreamcast
content:
- name: "ChuChu Rocket! (USA) (En,Ja,Fr,De,Es)"
catalog: redump
reason: >-
A GD-ROM that boots through the Dreamcast BIOS logo (dc_boot.bin) to
its title without input. Flycast runs it with its HLE BIOS too, so the
boot test shows the difference the file makes.
region: north-america
load: cue
frames: 3600
frame_hash_every: 10
nintendo-ds:
romfetch: nds
content:
- name: "Mario Kart DS (Japan)"
catalog: no-intro
reason: >-
One game in the three regions, each held by a romfetch source (the
Japanese Tetris DS is not). melonDS and DeSmuME boot with the
ARM7/ARM9 BIOS and firmware or with their free replacements, and
BizHawk ties firmware dumps to regions.
region: japan
frames: 3600
frame_hash_every: 10
- name: "Mario Kart DS (USA, Australia) (En,Fr,De,Es,It)"
catalog: no-intro
reason: >-
The US release of the same game, for the US firmware dump BizHawk
declares and for the boot test of bios7.bin, bios9.bin and
firmware.bin in melonDS.
region: north-america
frames: 3600
frame_hash_every: 10
- name: "Mario Kart DS (Europe) (En,Fr,De,Es,It)"
catalog: no-intro
reason: >-
The European release of the same game, third leg of the region check
for the firmware dumps BizHawk ties to a region.
region: europe
frames: 3600
frame_hash_every: 10
nintendo-gba:
romfetch: gba
content:
- name: "Advance Wars (USA)"
catalog: no-intro
reason: >-
A 4 MB cartridge that shows the Game Boy Advance boot logo when
gba_bios.bin is given and skips it on the cores' HLE BIOS, so the
boot test sees the optional BIOS at work.
region: north-america
frames: 3600
frame_hash_every: 10
panasonic-3do:
romfetch: 3do
content:
- name: "Gex (USA, Europe)"
catalog: redump
reason: >-
A 3DO disc: Opera needs a 3DO BIOS (panafz10.bin and the others the
profile lists) for any game, and Gex shows the 3DO logo the BIOS
draws before the game.
region: north-america
also_regions: [europe]
load: cue
frames: 3600
frame_hash_every: 10
philips-cdi:
romfetch: cdi
content:
- name: "Hotel Mario (Europe)"
catalog: redump
reason: >-
A CD-i disc: the CD-i cores run nothing without the player's system
ROM (cdimono1.zip in MAME, the BIOS SAME CDi declares). A 366 MB disc
that starts its intro without input.
region: europe
load: cue
frames: 3600
frame_hash_every: 10
commodore-amiga-cd32:
romfetch: cd32
content:
- name: "Pinball Fantasies (Europe)"
catalog: redump
reason: >-
A 97 MB CD32 disc: PUAE needs the CD32 Kickstart and extended ROM to
boot any CD32 title, and the disc shows the CD32 boot animation the
ROM draws.
region: europe
load: cue
frames: 3600
frame_hash_every: 10
atari-lynx:
romfetch: lynx
content:
- name: "Super Skweek (World) (Atari Lynx Collection 1) (Unl)"
catalog: no-intro
reason: >-
Handy and Beetle Lynx refuse to start a cartridge without
lynxboot.img. The headered Lynx catalog romfetch reads holds only the
Atari Lynx Collection releases and betas, and this is a full game
from that collection.
region: north-america
also_regions: [europe, japan]
frames: 3600
frame_hash_every: 10
# Everything else with a BIOS entry.
sega-mastersystem:
romfetch: mastersystem
content: &mastersystem
- name: "Alex Kidd no Miracle World (Japan)"
catalog: no-intro
reason: >-
The Japanese Mark III cartridge. Genesis Plus GX ties bios_J.sms to
Japan and bios_U.sms to the other regions, so the region check runs
the same game on each.
region: japan
frames: 3600
frame_hash_every: 10
- name: "Alex Kidd in Miracle World (USA, Europe)"
catalog: no-intro
reason: >-
The western cartridge of the same game, one release for the US and
Europe, as bios_U.sms serves both in the Genesis Plus GX profile.
region: north-america
also_regions: [europe]
frames: 3600
frame_hash_every: 10
# ares names the same console master-system.
master-system:
romfetch: mastersystem
content: *mastersystem
nintendo-64:
romfetch: n64
content:
- name: "Super Mario 64 (Japan)"
catalog: no-intro
reason: >-
One game in the three regions. simple64 loads the NTSC or the PAL PIF
ROM by the cartridge's region, which the region check proves on the
Japanese, US and European releases.
region: japan
frames: 3600
frame_hash_every: 10
- name: "Super Mario 64 (USA)"
catalog: no-intro
reason: >-
The US release of the same game, NTSC like the Japanese one, so it
should read the same PIF ROM as the Japanese cartridge.
region: north-america
frames: 3600
frame_hash_every: 10
- name: "Super Mario 64 (Europe) (En,Fr,De)"
catalog: no-intro
reason: >-
The PAL release of the same game: the one cartridge the PAL PIF ROM
(pif.pal) is for in simple64.
region: europe
frames: 3600
frame_hash_every: 10
nintendo-gamecube:
romfetch: gamecube
content:
- name: "Pikmin (Japan) (Rev 1)"
catalog: redump
reason: >-
One game in the three regions. Dolphin reads the IPL of the disc's
region (GC/JAP/IPL.bin) to show the GameCube boot animation, and
skips it on its HLE boot. 1.4 GB, fetched only when a check needs it.
region: japan
frames: 3600
frame_hash_every: 10
- name: "Pikmin (USA)"
catalog: redump
reason: >-
The US release of the same game, for GC/USA/IPL.bin in Dolphin and
its forks.
region: north-america
frames: 3600
frame_hash_every: 10
- name: "Pikmin (Europe, Australia) (En,Fr,De,Es,It)"
catalog: redump
reason: >-
The PAL release of the same game, for GC/EUR/IPL.bin. Third leg of
the region check.
region: europe
frames: 3600
frame_hash_every: 10
sega-megadrive:
romfetch: megadrive
content:
- name: "Sonic The Hedgehog (USA, Europe)"
catalog: no-intro
reason: >-
A cartridge that passes the TMSS check of a later Mega Drive: with
bios_MD.bin the core shows the TMSS licence screen first, without it
the game starts at once.
region: north-america
also_regions: [europe]
frames: 3600
frame_hash_every: 10
sega-gamegear:
romfetch: gamegear
content:
- name: "Sonic The Hedgehog (World) (Rev A)"
catalog: no-intro
reason: >-
A cartridge for the optional Game Gear boot ROM (bios.gg): with it the
core shows the Sega licence screen before the game.
region: north-america
frames: 3600
frame_hash_every: 10
nintendo-gb:
romfetch: gb
content:
- name: "Tetris (World) (Rev 1)"
catalog: no-intro
reason: >-
The cartridge every Game Boy core runs: with the optional boot ROM
(gb_bios.bin, dmg_boot.bin) the Nintendo logo scrolls down first,
without it the game starts at once.
region: north-america
frames: 3600
frame_hash_every: 10
nintendo-gbc:
romfetch: gbc
content:
- name: "Tetris DX (World) (SGB Enhanced) (GB Compatible)"
catalog: no-intro
reason: >-
A Game Boy Color cartridge for the optional CGB boot ROM
(gbc_bios.bin, cgb_boot.bin): the boot animation and the palette it
picks show whether the core read it.
region: north-america
frames: 3600
frame_hash_every: 10
snk-neogeo-pocket:
romfetch: ngp
content:
- name: "King of Fighters R-1 - Pocket Fighting Series (Japan, Europe) (En,Ja)"
catalog: no-intro
reason: >-
A Neo Geo Pocket cartridge for the optional BIOS some cores read (the
SNK boot screen and the clock setting it asks for). Mednafen's HLE
starts the game without it.
region: japan
also_regions: [europe]
frames: 3600
frame_hash_every: 10
nec-pc-fx:
romfetch: pcfx
content:
- name: "Battle Heat (Japan)"
catalog: redump
reason: >-
A PC-FX disc: Beetle PC-FX needs pcfx.rom for any game. Every PC-FX
release is Japanese.
region: japan
load: cue
frames: 3600
frame_hash_every: 10
sony-psp:
romfetch: psp
content:
- name: "Lumines - Puzzle Fusion (Europe) (En,Fr,De,Es,It)"
catalog: redump
reason: >-
A UMD image for PPSSPP, which needs no BIOS but reads its font and
asset files (the ppsspp data directory) to draw the system dialogs.
region: europe
frames: 3600
frame_hash_every: 10
# Systems with a BIOS entry that no catalog romfetch reads covers, or whose
# content romfetch cannot fetch. `romfetch` is what the request returned:
# its exit code and error code.
uncovered:
snk-neogeo-mvs:
want: "mslug (neogeo.zip with Metal Slug), through MAME and FBNeo"
reason: >-
Arcade sets are not in the Redump or No-Intro catalogs and romfetch has
no arcade system.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
sega-naomi:
want: "mvsc2 (naomi.zip with Marvel vs. Capcom 2), through MAME and Flycast"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
sammy-atomiswave:
want: "kofnw (awbios.zip with The King of Fighters Neowave), through MAME and Flycast"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
igs-pgm:
want: "kov (pgm.zip with Knights of Valour), through MAME and FBNeo"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
cps3:
want: "sfiii3 (Street Fighter III 3rd Strike), through MAME and FBNeo"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
sega-stv:
want: "shienryu (stvbios.zip with Shienryu), through MAME, FBNeo and Kronos"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
sega-triforce:
want: "a Triforce GD-ROM set, through Dolphin Triforce"
reason: Arcade set, no catalog romfetch reads.
romfetch: {exit: 1, code: unknown_system, request: "arcade"}
nintendo-dsi:
want: "a DSi-enhanced cartridge or DSiWare title, for melonDS in DSi mode"
reason: romfetch reads the decrypted DS catalog only, no DSi set.
romfetch: {exit: 1, code: unknown_system, request: "dsi"}
nintendo-3ds:
want: "one game per region, for Citra's regional system archives"
reason: No 3DS catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "3ds"}
nintendo-64dd:
want: "F-Zero X - Expansion Kit (Japan), for the 64DD IPL"
reason: No 64DD catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "n64dd"}
atari-jaguarcd:
want: "Myst (USA), for the Jaguar CD BIOS"
reason: No Jaguar CD catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "jaguarcd"}
intellivision:
want: "Astrosmash (USA, Europe), for exec.bin and grom.bin"
reason: No Intellivision catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "intellivision"}
colecovision:
want: "Donkey Kong (USA), for colecovision.rom"
reason: No ColecoVision catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "colecovision"}
msx:
want: "Gradius (Japan), for the MSX system ROMs"
reason: No MSX catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "msx"}
commodore-amiga:
want: "Lemmings (Europe), for the Kickstart ROMs"
reason: >-
romfetch reads the Amiga CD32 catalog only, no Amiga floppy catalog.
romfetch: {exit: 1, code: unknown_system, request: "amiga"}
atari-5200:
want: "Pac-Man (USA), for 5200.rom"
reason: No Atari 5200 catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "atari5200"}
atari-7800:
want: "Asteroids (USA), for the 7800 BIOS"
reason: No Atari 7800 catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "atari7800"}
nintendo-fds:
want: "Super Mario Brothers 2 (Japan), for disksys.rom"
reason: No Famicom Disk System catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "fds"}
nintendo-pokemon-mini:
want: "Pokemon Pinball Mini (USA, Europe), for bios.min"
reason: No Pokemon Mini catalog in romfetch.
romfetch: {exit: 1, code: unknown_system, request: "pokemini"}
microsoft-xbox:
want: "Halo - Combat Evolved (USA), for xemu's MCPX ROM, BIOS and hard disk image"
reason: >-
romfetch plans the 7.8 GB entry but no source holds it (exit 1, no
candidate), and it is above romfetch's 4 GiB limit that needs --yes.
romfetch: {exit: 1, code: null, request: "xbox"}
sony-playstation-3:
want: "Ridge Racer 7 (USA), for RPCS3's PS3UPDAT.PUP firmware"
reason: romfetch plans the entry but no source holds it (exit 1, no candidate).
romfetch: {exit: 1, code: null, request: "ps3"}
tests/campaign/oracle.yml
+500
View File
@@ -0,0 +1,500 @@
# The boot oracle of scripts/core_probe.py: what a run must reach to boot,
# and what tells a BIOS error apart.
#
# content per piece of test content (by the sha1 of the file handed to
# the core, the .cue for a disc), per core (by the library name
# retro_get_system_info gives): the reference frames a run with
# the full BIOS set reaches. Each was recorded once
# (`recorded_from`, the record id) and reviewed by looking at its
# PNG, from the variant `variant` names when that run was a
# check. `stage: bios` is a screen only the console's own BIOS
# draws, `stage: game` a screen of the content. A run that reaches
# one of them boots.
# error_patterns log lines after which a core stops, read from its source
# (`source_ref`). Shared ones at the top level, a core's own under
# cores:. A line a core logs and then carries on (a fallback to a
# built-in BIOS) is not an error pattern.
# error_screens per core, the sha1 of a frame that shows its BIOS error message.
# bios_patterns per core, log lines by which it says it loaded the console's
# own BIOS: they stand for the bios-stage screens of a core
# that hides them by default.
#
# A run that reaches no reference frame, matches no pattern or screen and
# does not end on one colour is `unknown`: frames that differ from another
# run's never make a boot.
error_patterns: []
cores:
# The PlayStation cores below log a missing or wrong BIOS and boot a
# built-in one: none of those lines is an error pattern.
"Beetle PSX":
note: >-
A missing BIOS logs "Firmware is missing" and loads the embedded
OpenBIOS, a wrong one logs "Firmware found but has invalid SHA1" and is
loaded anyway (libretro.c:849, 856, 3504-3511 at 7fea57a5).
error_patterns: []
error_screens:
- sha1: dfbfc299250c0983f86bbf951efd187441306d92
shows: the Japanese BIOS main menu, the disc refused (a US disc under a Japanese BIOS)
"Beetle PSX HW":
note: The same code as Beetle PSX, built with the hardware renderers.
error_patterns: []
error_screens:
- sha1: dfbfc299250c0983f86bbf951efd187441306d92
shows: the Japanese BIOS main menu, the disc refused (a US disc under a Japanese BIOS)
"SwanStation":
note: >-
"No BIOS image found for <region> region" is followed by the built-in
OpenBIOS (src/core/host_interface.cpp:182, src/core/system.cpp:678-682
at 631f0fb3).
error_patterns: []
error_screens:
- sha1: 167fb7b6527e295b12dca5806a29856f5f31d401
shows: the Japanese BIOS main menu, the disc refused (a US disc under a Japanese BIOS)
"PCSX-ReARMed":
note: >-
Without a BIOS, or with one of the wrong size, it logs "Using HLE BIOS"
and runs its own (frontend/libretro.c:3893-3899 at c8816799). The BIOS
boot logo is off by default (pcsx_rearmed_show_bios_bootlogo), so its
runs reach no bios-stage screen: the line it logs once a file passes
its signature check and is kept stands for them.
bios_patterns:
- pattern: '^found (US|EU|JP) BIOS file, crc32 '
source_ref: frontend/libretro.c:3646-3669 at c8816799
error_patterns: []
error_screens: []
"Beetle Saturn":
note: Without a usable BIOS the core logs one of these and retro_load_game fails.
error_patterns:
- pattern: '^Cannot open BIOS file '
source_ref: mednafen/ss/ss.c:1943-1947 at 2ce77e20
- pattern: '^BIOS file ".*" is of an incorrect size\.'
source_ref: mednafen/ss/ss.c:1961-1967 at 2ce77e20
- pattern: '^BIOS file ".*" could not be fully read'
source_ref: mednafen/ss/ss.c:1978-1983 at 2ce77e20
error_screens: []
"pcsx2":
note: >-
PCSX2 standalone (scripts/standalone_probe.py). Without a BIOS it shows a
dialog and runs nothing: the dialog is its error screen, captured as the
1280x720 root window.
error_patterns: []
error_screens:
- sha1: 46329c633ea7d0b63349261e4aab169e4f44d3f6
shows: the dialog "PCSX2 requires a PlayStation 2 BIOS in order to run"
"PCSX1":
note: An HLE BIOS stands in when none is found, and the run logs nothing about it.
error_patterns: []
error_screens: []
content:
- name: "Bust-A-Move 2 - Arcade Edition (USA)"
catalog: redump
# The sha1 of the .cue handed to the core.
sha1: ed4a437a6f653cc71ad6ac0da68cc85eb093f298
references:
"Beetle PSX":
recorded_from: 2026-10-27-b0150a15670d
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 0905c920a1de996664eddca94f93f75d0dcd4c52
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment America
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: 63a96bd6d1d487da1824e10d37f6658506edc451
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: 6628416f377d2e25df45e5fa2126f8f9313551f0
stage: game
shows: the Acclaim logo, CREDIT 6
- sha1: d68892362e0f33c41cc59dac1b5a2077fe5428ee
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 56330c6c7646c2c261acb3f48450825cb289845a
stage: game
shows: a black screen with CREDIT 6
"Beetle PSX HW":
recorded_from: 2026-10-27-ee69ad83c9f8
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 0905c920a1de996664eddca94f93f75d0dcd4c52
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment America
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: 63a96bd6d1d487da1824e10d37f6658506edc451
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: 6628416f377d2e25df45e5fa2126f8f9313551f0
stage: game
shows: the Acclaim logo, CREDIT 6
- sha1: d68892362e0f33c41cc59dac1b5a2077fe5428ee
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 56330c6c7646c2c261acb3f48450825cb289845a
stage: game
shows: a black screen with CREDIT 6
"SwanStation":
recorded_from: 2026-10-27-717f780131fc
frames:
- sha1: fad859d9b8211ce9f35ad3a05b0b7d184fe14d83
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 593a6e2600946040fefd6eb9aebdc5c0d56f0555
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment America
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: 533715dd946ba1984af1b9356e06989c7770e04a
stage: game
shows: the Acclaim logo, CREDIT 6
- sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
stage: game
shows: a black screen with CREDIT 6
"PCSX-ReARMed":
recorded_from: 2026-10-27-8154bab7bf80
frames:
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: 533715dd946ba1984af1b9356e06989c7770e04a
stage: game
shows: the Acclaim logo, CREDIT 6
- sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
stage: game
shows: a black screen with CREDIT 6
"PCSX1":
recorded_from: 2026-10-27-2905a9dd4b0a
frames:
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: e10f642259f2906bb39ae07fcb4b5a36eae18287
stage: game
shows: the Acclaim logo with a stray strip at its right edge, CREDIT 6
- sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
stage: game
shows: a black screen with CREDIT 6
"duckstation":
recorded_from: 2026-10-27-9bc8fbcebc85
variant: scph5501.bin
frames:
- sha1: 7218f6facf5c27248b8dc9cff31d8b4aeb3b1829
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: d70b0e9e9e4e05eb54fb56d6adbab3afb753631a
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment America
- sha1: f336e067bb644abaf2ab30462564a1baa037d58e
stage: game
shows: Now Loading with the yellow bubble
- sha1: 534459d74bac4644fe7d3f23689390026bbb6d1b
stage: game
shows: the Acclaim and Taito credits text, CREDIT 6
- sha1: b01b0ebfe12f3bcfd7250443f391a67f002c40d4
stage: game
shows: the Acclaim logo, CREDIT 6
- sha1: 5d83577be2b2ca48bd55daaede7cc4ba96b7fee7
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 4b4e686aeae9fecbb8cb504dcddb9b91bc1a137e
stage: game
shows: a black screen with CREDIT 6
- sha1: 2901974f4a50e0a9f061b21e5f262ea7edf5b8c2
stage: game
shows: the attract-mode game, two playfields of bubbles
- name: "Puzzle Bobble 2 (Japan)"
catalog: redump
sha1: 3b51b45b8b78777c04086fcd1ef1dd0439c365e6
references:
"Beetle PSX":
recorded_from: 2026-10-27-a2bd916f1765
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 1d5305661b110a78aff8e41d60c19168c4af77ac
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Inc.
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: 80d023ecbf776a97b6bb2bdeb13e43460f60fd53
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 1bda114b66fd098623abca772cf23813293a3849
stage: game
shows: a black screen with CREDIT 6
- sha1: 3bb53b8a3036554a4bf9ec4785df7a4b4e06a87d
stage: game
shows: the attract-mode game on the beach, PASSAGE TIME IS
"Beetle PSX HW":
recorded_from: 2026-10-27-6e0e946eeba5
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 1d5305661b110a78aff8e41d60c19168c4af77ac
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Inc.
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: 80d023ecbf776a97b6bb2bdeb13e43460f60fd53
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 1bda114b66fd098623abca772cf23813293a3849
stage: game
shows: a black screen with CREDIT 6
- sha1: 3bb53b8a3036554a4bf9ec4785df7a4b4e06a87d
stage: game
shows: the attract-mode game on the beach, PASSAGE TIME IS
"PCSX-ReARMed":
recorded_from: 2026-10-27-65923ebc1836
frames:
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: 2443e73be7ce3203408d7398c997fe6ec7cebd9f
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 373e5e3d7c524ed1556b9292a794169bf1de27d0
stage: game
shows: a black screen with CREDIT 6
- sha1: 53f32c4acd5a522f359952ee48f138e44aee6e61
stage: game
shows: the attract-mode game on the beach, PASSAGE TIME IS
"SwanStation":
recorded_from: 2026-10-27-abe38e414339
frames:
- sha1: fad859d9b8211ce9f35ad3a05b0b7d184fe14d83
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 9c168a8d6a6c706157307910e2b46039d019e48b
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Inc.
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: 2443e73be7ce3203408d7398c997fe6ec7cebd9f
stage: game
shows: the Taito logo on white, CREDIT 6
- sha1: 373e5e3d7c524ed1556b9292a794169bf1de27d0
stage: game
shows: a black screen with CREDIT 6
- sha1: 53f32c4acd5a522f359952ee48f138e44aee6e61
stage: game
shows: the attract-mode game on the beach, PASSAGE TIME IS
- name: "Bust-A-Move 2 - Arcade Edition (Europe)"
catalog: redump
sha1: fbfb904cdb7a0e25a659980a5ed1206298bc7a1e
references:
"Beetle PSX":
recorded_from: 2026-10-27-4b9e700d03a9
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 408effe5ed13b50c97a21d2efe9930ab0d2b6f43
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Europe
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: f5649c8df958220324753968e8785ea80a021837
stage: game
shows: the anti-piracy notice
- sha1: 2cccff1e0706fcc8bdd4fdcd35e0310c9aaf9ac5
stage: game
shows: the Acclaim and Taito credits text
- sha1: dd70e8e057c3de89f0c39c2b91b72fce902a8588
stage: game
shows: the Acclaim logo
- sha1: 368ac3ce2a4ef7dcc1720e0e48f18e3e6404e191
stage: game
shows: the Taito logo on white
"Beetle PSX HW":
recorded_from: 2026-10-27-f3b61598f415
frames:
- sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: 408effe5ed13b50c97a21d2efe9930ab0d2b6f43
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Europe
- sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
stage: game
shows: Now Loading with the yellow bubble
- sha1: f5649c8df958220324753968e8785ea80a021837
stage: game
shows: the anti-piracy notice
- sha1: 2cccff1e0706fcc8bdd4fdcd35e0310c9aaf9ac5
stage: game
shows: the Acclaim and Taito credits text
- sha1: dd70e8e057c3de89f0c39c2b91b72fce902a8588
stage: game
shows: the Acclaim logo
- sha1: 368ac3ce2a4ef7dcc1720e0e48f18e3e6404e191
stage: game
shows: the Taito logo on white
"PCSX-ReARMed":
recorded_from: 2026-10-27-c71f71b579b9
frames:
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: d0c5e568fd2585ad5c0ed244c25bdacdbb596d55
stage: game
shows: the anti-piracy notice
- sha1: 4ea5612d272a8b05fabf98ed7dc4b245fe759001
stage: game
shows: the Acclaim and Taito credits text
- sha1: ed433c340dc7ef65c8afba4434542c1827251127
stage: game
shows: the Acclaim logo
- sha1: a909b761d6f46ec28bdf46aac874873db40396b7
stage: game
shows: the Taito logo on white
"SwanStation":
recorded_from: 2026-10-27-57e092562c10
frames:
- sha1: fad859d9b8211ce9f35ad3a05b0b7d184fe14d83
stage: bios
shows: the Sony Computer Entertainment logo
- sha1: e6f4f8524f3c8a47d5dba0d33f63c64dcebccde7
stage: bios
shows: the PlayStation logo licensed by Sony Computer Entertainment Europe
- sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
stage: game
shows: Now Loading with the yellow bubble
- sha1: d0c5e568fd2585ad5c0ed244c25bdacdbb596d55
stage: game
shows: the anti-piracy notice
- sha1: 4ea5612d272a8b05fabf98ed7dc4b245fe759001
stage: game
shows: the Acclaim and Taito credits text
- sha1: ed433c340dc7ef65c8afba4434542c1827251127
stage: game
shows: the Acclaim logo
- sha1: a909b761d6f46ec28bdf46aac874873db40396b7
stage: game
shows: the Taito logo on white
- name: "Puzzle Bobble 2X (Japan) (2M1, 2MB1)"
catalog: redump
sha1: b962ec9148a189239810a10677dcba073983733f
references:
"Beetle Saturn":
recorded_from: 2026-10-27-ea155398ce4b
frames:
- sha1: 5edec5180d1c04dc0ead1e5fbbc7d6d53cc23fdb
stage: bios
shows: the SEGA SATURN logo of the Japanese BIOS
- sha1: dcd45679fcd31a423d0e54690e89b52f417d8728
stage: bios
shows: SEGA, produced by or under license from Sega Enterprises
- sha1: 7efe0a96245086d186a702a253525614d03f01f4
stage: game
shows: NOW LOADING
- sha1: 79894146225fefecb7db22b6aa296cb035a8d8ae
stage: game
shows: the Taito logo on white, CREDIT 9
- sha1: 475fce75c7c581b7367ac1a4adaa0aadc18afab1
stage: game
shows: GAME OVER, press 1P or 2P start button
- sha1: f610c613f723fa640250121d6086b5e2c53b695c
stage: game
shows: the Puzzle Bobble 2X title, press start button
- name: "Bust-A-Move 2 - Arcade Edition (USA)"
catalog: redump
sha1: 782b93d04ecb9da4cc534489e67feeac7ea9dc78
references:
"Beetle Saturn":
recorded_from: 2026-10-27-35dd5587262d
frames:
- sha1: f6f6ad7714da4b7c32a7736276c8cba7ef9d45d0
stage: bios
shows: the SEGA SATURN globe logo
- sha1: dcd45679fcd31a423d0e54690e89b52f417d8728
stage: bios
shows: SEGA, produced by or under license from Sega Enterprises
- sha1: e0b0fd36c4da2579b509934b8fe010f4741edfa5
stage: game
shows: NOW LOADING
- sha1: 97f62d0835bc7f27b660f330d794c149c50c5716
stage: game
shows: the Acclaim and Taito credits text, CREDIT 9
- sha1: 0cd323a8a93adec3a2173549300acbf9a3fe5596
stage: game
shows: the Acclaim logo, CREDIT 9
- sha1: 76ed20462b9f45e247611fa77feff69e47052875
stage: game
shows: the Taito logo on white, CREDIT 9
- sha1: e90793d79a1c815cd74e7a56f0e9d8f124bfc547
stage: game
shows: the Bust-A-Move 2 title, press start button
- name: "Bust-A-Move 2 - Arcade Edition (Europe)"
catalog: redump
sha1: 9298fabcdd797272c8c993abc81fcdd51a4e7830
references:
"Beetle Saturn":
recorded_from: 2026-10-27-1880da25f150
frames:
- sha1: 2d870b2352923df4b2563f75c8a01a3d6dcb4237
stage: bios
shows: the SEGA SATURN globe logo
- sha1: 95410b3a4a9f69ae981e2774c55574886efa67ea
stage: bios
shows: SEGA, produced by or under license from Sega Enterprises
- sha1: 4d426aec90be9bff76e2d32da0437d91f9880950
stage: game
shows: NOW LOADING
- sha1: 5bfbc97567caa32bad7489acc6c9a4f3f65ed863
stage: game
shows: the Acclaim and Taito credits text, CREDIT 9
- sha1: 8af02ae2f2f0864657a3c7a68293a4da163000d6
stage: game
shows: the Acclaim logo, CREDIT 9
- sha1: e3245a31197fa028c2da63f387f58a38e010c98d
stage: game
shows: the Taito logo on white, CREDIT 9
- sha1: 9ae4dbf2ded891887635634d224dedccf176c1a0
stage: game
shows: the Bust-A-Move 2 title, press start button
- name: "Rez (USA) (En,Fr,De,Es,It)"
catalog: redump
sha1: f6ac65761e9d8c8c9eb9d3377ff53719557ae346
references:
"pcsx2":
recorded_from: 2026-10-27-fac0b52c9475
variant: ps2-0230a-20080220.bin
frames:
- sha1: 003685f698160f1cc79ca0bc00091f30ccb681db
stage: game
shows: Rez asking to format the unformatted memory card in slot 1
tests/test_campaign.py
+282 -81
View File
@@ -16,6 +16,7 @@ import io
import json
import os
import shutil
import struct
import subprocess
import sys
import tempfile
@@ -92,15 +93,20 @@ void retro_init(void) {}
void retro_deinit(void) {}
static FILE *try_open(const char *dir, const char *name) {
char path[4096]; snprintf(path, sizeof path, "%s/%s", dir, name); return fopen(path, "rb"); }
/* booted: 0 a black screen to the end, 1 the BIOS logo then the title,
2 the title alone (the built-in stand-in of FAKE_HLE). */
bool retro_load_game(const void *g) {
const char *dir = NULL; FILE *f; (void)g; env(9, &dir);
const char *dir = NULL; FILE *f; unsigned char buf[8]; size_t n = 0; (void)g; env(9, &dir);
f = try_open(dir, "bios.bin");
if (!f) f = try_open(dir, "BIOS.BIN");
if (f) { n = fread(buf, 1, sizeof buf, f); fclose(f); }
int valid = f && n == 4 && buf[0] == 'B';
#ifdef FAKE_HLE
if (f) fclose(f);
booted = 1;
/* Checks the size only: a file of the wrong size is refused for the
stand-in, one of the right size is run, whatever it holds. */
booted = valid ? 1 : (f && n == 4) ? 0 : 2;
#else
if (f) { fclose(f); booted = 1; } else booted = 0;
booted = valid ? 1 : 0;
#endif
return true; }
void retro_unload_game(void) {}
@@ -109,7 +115,8 @@ void retro_run(void) {
#ifdef FAKE_HANG
for (;;) {}
#endif
for (int i = 0; i < 8; i++) pixels[i] = booted ? (uint16_t)(frame / 2 + i) : 0;
int shown = frame < 4 ? 0 : booted == 1 && frame < 8 ? 1 : booted ? 2 : 0;
for (int i = 0; i < 8; i++) pixels[i] = shown == 0 ? 0 : (uint16_t)((shown == 1 ? 1 : 20) + i);
frame++;
video(pixels, 4, 2, 8); batch(s, 2); }
size_t retro_serialize_size(void) { return 0; }
@@ -117,78 +124,103 @@ bool retro_serialize(void *d, size_t n) { (void)d; (void)n; return false; }
"""
def rep(frames: list[str], uniform: list[str] | None = None, loaded: bool = True) -> dict:
return {"loaded": loaded, "observables": {
"frame_hashes": [[i * 10, h] for i, h in enumerate(frames)],
"uniform_frames": sorted(set(uniform or []))}}
def run(verdict: str, label: str = "v", stages: tuple[str, ...] = (),
rid: str = "2026-10-27-aaaaaaaaaaaa") -> rules.Run:
return rules.Run(verdict, rid, label, stages)
def boot_record(absent: list[dict], present: list[dict], rid: str = "2026-10-27-aaaaaaaaaaaa") -> dict:
return {"id": rid, "variants": [{"label": "bios.bin-absent", "repetitions": absent},
{"label": "bios.bin", "repetitions": present}]}
class BootOutcome(unittest.TestCase):
GAME = ("black", "logo", "title", "title2")
def test_boots_when_screens_are_shared(self) -> None:
record = boot_record([rep(["black", "hle", "logo", "title"], ["black"])] * 2,
[rep(self.GAME, ["black"])] * 2)
boot = rules.boot_outcome(record, "bios.bin-absent", "bios.bin")
self.assertEqual((boot.status, boot.shared), (rules.BOOTS, 2))
def test_one_colour_and_no_load_fail(self) -> None:
black = boot_record([rep(["black"], ["black"]), rep([], [], loaded=False)],
[rep(self.GAME, ["black"])])
self.assertEqual(rules.boot_outcome(black, "bios.bin-absent", "bios.bin").status, rules.FAILS)
def test_a_shared_blank_screen_proves_nothing(self) -> None:
# The black frame both runs start on is one colour: it is no game screen.
record = boot_record([rep(["black", "error"], ["black"])], [rep(self.GAME, ["black"])])
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
rules.INCONCLUSIVE)
def test_one_shared_screen_is_not_enough(self) -> None:
record = boot_record([rep(["logo", "error"])], [rep(self.GAME, ["black"])])
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
rules.INCONCLUSIVE)
def test_repetitions_that_disagree_settle_nothing(self) -> None:
record = boot_record([rep(["logo", "title"]), rep(["black"], ["black"])],
[rep(self.GAME, ["black"])])
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
rules.INCONCLUSIVE)
def test_a_reference_that_shows_nothing_settles_nothing(self) -> None:
record = boot_record([rep(["logo", "title"])], [rep(["black"], ["black"])])
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
rules.INCONCLUSIVE)
record = boot_record([rep(["logo", "title"])], [rep(self.GAME, loaded=False)])
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
rules.INCONCLUSIVE)
NONE_BOOTS = run(rules.BOOTS, "none", ("game",), "2026-10-27-000000000001")
NONE_HANGS = run(rules.HANG, "none", (), "2026-10-27-000000000001")
PRESENT = run(rules.BOOTS, "bios.bin", ("bios", "game"), "2026-10-27-000000000002")
class RuleTable(unittest.TestCase):
ENTRY = MappingProxyType({"name": "bios.bin", "required": True, "aliases": ["Bios.bin"]})
RID = "2026-10-27-bbbbbbbbbbbb"
def test_boots_without_settles_hle_and_queues_required(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3), self.ENTRY, self.RID)
def test_a_verdict_is_read_from_the_record(self) -> None:
record = {"id": self.RID, "variants": [
{"label": "a", "verdict": "boots", "verdict_evidence": {"rule": "repetitions_agree",
"detail": "d", "stages": ["bios"]}},
{"label": "old"}]}
self.assertEqual(rules.run_of(record, "a"), rules.Run("boots", self.RID, "a", ("bios",)))
# A record from before the oracle proves nothing.
self.assertEqual(rules.run_of(record, "old").verdict, rules.UNKNOWN)
def test_boots_with_no_bios_settles_hle_and_queues_required(self) -> None:
verdicts = rules.rule_need(NONE_BOOTS, PRESENT, self.ENTRY)
self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
[("hle_fallback", True, True), ("required", False, False)])
self.assertEqual(verdicts[0].evidence, (self.RID,))
optional = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3),
{"name": "a", "required": False}, self.RID)
self.assertEqual(verdicts[0].evidence, (NONE_BOOTS.record_id,))
optional = rules.rule_need(NONE_BOOTS, PRESENT, {"name": "a", "required": False})
self.assertEqual([v.field for v in optional], ["hle_fallback"])
def test_fails_without_settles_required_and_no_hle(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "x"), self.ENTRY, self.RID)
self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
[("required", True, True), ("hle_fallback", False, True)])
def test_a_file_the_content_never_uses_settles_nothing(self) -> None:
unused = run(rules.BOOTS, "bios.bin", ("game",))
self.assertFalse(rules.read(NONE_BOOTS, unused, unused, unused))
self.assertEqual(rules.rule_need(NONE_BOOTS, unused, self.ENTRY, used=False), [])
# A wrong file that hangs where no file boots: the core reads it.
self.assertTrue(rules.read(NONE_BOOTS, unused, run(rules.HANG), unused))
self.assertTrue(rules.read(NONE_BOOTS, PRESENT, None, None))
def test_inconclusive_is_queued(self) -> None:
(verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "x"), self.ENTRY, self.RID)
self.assertFalse(verdict.apply)
def test_fails_without_and_boots_alone_settles_required(self) -> None:
for failure in rules.FAILS:
verdicts = rules.rule_need(run(failure, "none"), PRESENT, self.ENTRY)
self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
[("required", True, True), ("hle_fallback", False, True)])
def test_unknown_or_no_boot_alone_settles_nothing(self) -> None:
for none, present in ((run(rules.UNKNOWN), PRESENT), (NONE_HANGS, run(rules.HANG)),
(NONE_HANGS, run(rules.UNKNOWN))):
(verdict,) = rules.rule_need(none, present, self.ENTRY)
self.assertFalse(verdict.apply)
self.assertEqual(verdict.rule, "unsettled")
def test_file_use(self) -> None:
self.assertEqual(rules.file_use(run(rules.BIOS_ERROR), NONE_HANGS, PRESENT), rules.REFUSED)
# It boots without the BIOS screens the right file shows: a fallback.
self.assertEqual(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, PRESENT),
rules.REFUSED)
self.assertEqual(rules.file_use(run(rules.BOOTS, stages=("bios", "game")), NONE_BOOTS, PRESENT),
rules.USED)
self.assertEqual(rules.file_use(run(rules.HANG), NONE_BOOTS, PRESENT), rules.USED)
self.assertEqual(rules.file_use(run(rules.CRASH), NONE_BOOTS, PRESENT), rules.USED)
# A hang the run with no file shows too says nothing about the file.
self.assertIsNone(rules.file_use(run(rules.HANG), NONE_HANGS, PRESENT))
self.assertIsNone(rules.file_use(run(rules.UNKNOWN), NONE_BOOTS, PRESENT))
self.assertIsNone(rules.file_use(run(rules.BIOS_ERROR), NONE_BOOTS, run(rules.HANG)))
no_logo = run(rules.BOOTS, stages=("game",))
self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, no_logo))
def test_validation_agrees_or_adds_size(self) -> None:
refused = run(rules.BOOTS, stages=("game",))
used = run(rules.HANG)
entry = {"name": "bios.bin", "validation": ["size", "md5"]}
verdict = rules.rule_validation(refused, refused, NONE_BOOTS, PRESENT, entry)
self.assertEqual((verdict.rule, verdict.apply), ("validation_agrees", True))
self.assertTrue(rules.agrees(entry, verdict))
added = rules.rule_validation(used, refused, NONE_BOOTS, PRESENT, {"name": "bios.bin"})
self.assertEqual((added.rule, added.value, added.apply), ("validation_size_added", ["size"], True))
self.assertFalse(rules.agrees({"name": "bios.bin"}, added))
self.assertEqual(rules.rule_validation(used, used, NONE_BOOTS, PRESENT, {"name": "b"}).rule,
"validation_agrees")
def test_validation_contradicted_is_queued(self) -> None:
refused = run(rules.BIOS_ERROR)
used = run(rules.HANG)
cases = [
(refused, None, {"name": "b", "validation": ["size"]}), # content check undeclared
(used, None, {"name": "b", "validation": ["md5"]}), # declared, not done
(None, used, {"name": "b", "validation": ["size"]}), # half of it runs
(used, refused, {"name": "b", "validation": {"core": ["md5"]}}), # per-build form
]
for zeros, half, entry in cases:
verdict = rules.rule_validation(zeros, half, NONE_BOOTS, PRESENT, entry)
self.assertEqual((verdict.rule, verdict.apply), ("validation_contradicted", False), entry)
self.assertEqual(set(verdict.value), {"content", "size"})
self.assertFalse(rules.agrees(entry, verdict))
self.assertIsNone(rules.rule_validation(run(rules.UNKNOWN), None, NONE_BOOTS, PRESENT,
{"name": "b"}))
def test_path_and_alias(self) -> None:
self.assertIsNone(rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("bios.bin", self.RID)))
@@ -204,8 +236,14 @@ class RuleTable(unittest.TestCase):
self.assertEqual(verdict.rule, "never_opened")
def test_variants(self) -> None:
self.assertIsNone(rules.rule_variants("same", ["a", "b"], self.ENTRY, self.RID))
self.assertFalse(rules.rule_variants("differs", ["b", "a"], self.ENTRY, self.RID).apply)
self.assertIsNone(rules.rule_variants([run(rules.BOOTS, "a"), run(rules.BOOTS, "b")], self.ENTRY))
verdict = rules.rule_variants([run(rules.BOOTS, "b"), run(rules.HANG, "a")], self.ENTRY)
self.assertEqual((verdict.value, verdict.apply), (["a", "b"], False))
def test_every_settled_field_can_carry_evidence(self) -> None:
schema = json.loads((REPO_ROOT / "schemas" / "emulator.schema.json").read_text())
evidence = schema["properties"]["files"]["items"]["properties"]["evidence"]["properties"]
self.assertEqual(set(rules.SETTLED_FIELDS), set(evidence))
def test_agreement_and_evidence(self) -> None:
self.assertTrue(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", False, "", ())))
@@ -213,6 +251,22 @@ class RuleTable(unittest.TestCase):
self.assertTrue(rules.agrees(self.ENTRY, rules.Verdict("r", "aliases", ["Bios.bin"], "", ())))
merged = rules.merge_evidence({"required": ["x"]}, rules.Verdict("r", "required", True, "", ("y", "x")))
self.assertEqual(merged, {"required": ["x", "y"]})
# An id a frame comparison wrote no longer proves the field.
merged = rules.merge_evidence({"required": ["old"], "path": ["p"]},
rules.Verdict("r", "required", True, "", ("new",)),
keep=lambda rid: rid != "old")
self.assertEqual(merged, {"path": ["p"], "required": ["new"]})
def test_reference_candidates(self) -> None:
def rep(hashes: list[str]) -> dict:
return {"observables": {"frame_hashes": [[i * 10, h] for i, h in enumerate(hashes)],
"uniform_frames": ["black"]}}
record = {"variants": [{"repetitions": [
rep(["black"] * 4 + ["logo"] * 3 + ["flash"] + ["title"] * 5),
rep(["black"] * 5 + ["logo"] * 3 + ["title"] * 3)]}]}
self.assertEqual(campaign.reference_candidates(record),
[{"sha1": "logo", "first_frame": 40, "samples": 3},
{"sha1": "title", "first_frame": 80, "samples": 5}])
class ProfileEditor(unittest.TestCase):
@@ -286,6 +340,11 @@ class FakeHost(campaign.Host):
return result
def screen(base: int) -> str:
"""The sha1 of a frame of FAKE_CORE: eight 16-bit pixels from base up."""
return hashlib.sha1(struct.pack("<8H", *range(base, base + 8))).hexdigest()
class Workspace(unittest.TestCase):
def setUp(self) -> None:
TMP.mkdir(parents=True, exist_ok=True)
@@ -309,9 +368,17 @@ class Workspace(unittest.TestCase):
"name": "Fake Game (World)", "catalog": "no-intro", "reason": "a test",
"frames": 12, "frame_hash_every": 2}]}}}), encoding="utf-8")
self.game = self.tmp / "content" / "fakesys" / "Fake Game (World)"
self.oracle = self.tmp / "oracle.yml"
self.oracle.write_text(yaml.safe_dump({
"error_patterns": [], "cores": {}, "content": [{
"name": "Fake Game (World)", "sha1": hashlib.sha1(b"GAME").hexdigest(),
"references": {"Fake": {"recorded_from": "2026-10-27-000000000000", "frames": [
{"sha1": screen(1), "stage": "bios", "shows": "the fake logo"},
{"sha1": screen(20), "stage": "game", "shows": "the fake title"}]}}}]}),
encoding="utf-8")
self.paths = campaign.Paths(root=self.root, home=self.tmp / "home",
content=self.tmp / "content", db=self.root / "database.json",
content_file=self.content_file)
content_file=self.content_file, oracle=self.oracle)
def tearDown(self) -> None:
shutil.rmtree(self.tmp, ignore_errors=True)
@@ -333,13 +400,30 @@ class Workspace(unittest.TestCase):
class StateAndResume(Workspace):
def test_should_run(self) -> None:
self.assertTrue(campaign.should_run(None, "a", "d"))
done = {"status": "done", "core_sha256": "a", "profile_digest": "d"}
done = {"status": "done", "core_sha256": "a", "profile_digest": "d",
"record_version": mr.RECORD_VERSION, "oracle_sha256": ""}
self.assertFalse(campaign.should_run(done, "a", "d"))
# A record from before the verdicts, or judged against another oracle, runs again.
self.assertTrue(campaign.should_run({**done, "record_version": 3}, "a", "d"))
self.assertTrue(campaign.should_run(done, "a", "d", "f" * 64))
self.assertTrue(campaign.should_run(done, "b", "d"))
self.assertTrue(campaign.should_run(done, "a", "e"))
self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d"))
self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d"))
def test_only_its_own_oracle_entries_make_a_check_run_again(self) -> None:
key = [hashlib.sha1(b"GAME").hexdigest(), "Fake"]
before = self.make().oracle_digest(key)
data = yaml.safe_load(self.oracle.read_text())
data["content"][0]["references"]["Other"] = {"recorded_from": "2026-10-27-000000000000",
"frames": [{"sha1": "a" * 40, "stage": "game",
"shows": "x"}]}
self.oracle.write_text(yaml.safe_dump(data), encoding="utf-8")
self.assertEqual(self.make().oracle_digest(key), before)
data["cores"] = {"Fake": {"error_patterns": [{"pattern": "BIOS", "reason": "r"}]}}
self.oracle.write_text(yaml.safe_dump(data), encoding="utf-8")
self.assertNotEqual(self.make().oracle_digest(key), before)
def test_evidence_does_not_change_the_digest(self) -> None:
profile = yaml.safe_load(PROFILE)
before = campaign.profile_digest(profile)
@@ -752,8 +836,14 @@ class Findings(Workspace):
def entry(self, index: int = 0) -> dict:
return yaml.safe_load(self.profile.read_text())["files"][index]
def none(self, verdict: str) -> rules.Run:
return rules.Run(verdict, self.RID, "none")
def present(self) -> rules.Run:
return rules.Run(rules.BOOTS, self.RID, "bios.bin", ("game",))
def test_a_settled_field_is_written_with_its_evidence(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
entry = self.entry()
self.assertEqual(entry["hle_fallback"], True)
@@ -767,7 +857,7 @@ class Findings(Workspace):
self.assertEqual(len(self.lines()), 2)
def test_an_agreeing_run_adds_evidence_only(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "nothing"), self.entry(), self.RID)
verdicts = rules.rule_need(self.none(rules.HANG), self.present(), self.entry())
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
entry = self.entry()
self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False))
@@ -775,13 +865,22 @@ class Findings(Workspace):
self.assertEqual(self.lines(), [])
def test_a_later_agreement_resolves_the_open_finding(self) -> None:
(verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "?"), self.entry(), self.RID)
(verdict,) = rules.rule_need(self.none(rules.UNKNOWN), self.present(), self.entry())
self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict])
self.assertEqual(self.lines()[-1]["status"], "open")
later = rules.Verdict("inconclusive", "hle_fallback", False, "agrees now", ("2026-10-28-000000000000",))
self.c.settle("fake", self.content, self.core, 0, self.entry(), [later])
self.assertEqual([line["status"] for line in self.lines()], ["open", "resolved"])
def test_new_evidence_writes_the_finding_again(self) -> None:
verdict = rules.Verdict("unsettled", "hle_fallback", None, "x", (self.RID,))
self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict])
self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict])
later = rules.Verdict("unsettled", "hle_fallback", None, "x", ("2026-10-28-000000000000",))
self.c.settle("fake", self.content, self.core, 0, self.entry(), [later])
self.assertEqual([line["evidence"] for line in self.lines()],
[[self.RID], ["2026-10-28-000000000000"]])
def test_never_opened_is_flagged(self) -> None:
verdict = rules.rule_never_opened(self.entry(1), [self.RID], "Fake Game (World)")
self.c.settle("fake", self.content, self.core, 1, self.entry(1), [verdict])
@@ -789,7 +888,7 @@ class Findings(Workspace):
self.assertNotIn("evidence", self.entry(1))
def test_finding_lines_validate(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
self.c.write_run_line(self.c.current)
try:
@@ -802,7 +901,7 @@ class Findings(Workspace):
self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [], name)
def test_the_batch_commit_plan(self) -> None:
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
self.c.current.touched.add(mr.records_path("fake"))
plan = self.c.plan_commits(set())
@@ -819,6 +918,37 @@ class Findings(Workspace):
self.assertIn("would commit 'chore: add campaign measurements'", out.getvalue())
class ContentFile(Workspace):
def test_a_content_list_can_be_named(self) -> None:
other = self.tmp / "other.yml"
other.write_text(yaml.safe_dump({"systems": {"fake-system": {
"romfetch": "fakesys", "content": [{"name": "Other Game (Japan)", "catalog": "redump",
"reason": "r"}]}}}), encoding="utf-8")
out = io.StringIO()
with unittest.mock.patch.object(campaign.Paths, "default", return_value=self.paths), \
contextlib.redirect_stdout(out):
self.assertEqual(campaign.main(["run", "--dry-run", "--content-file", str(other)]), 0)
self.assertIn("Other Game (Japan)", out.getvalue())
class InstallTimer(unittest.TestCase):
def test_writes_the_units_and_enables_only_when_asked(self) -> None:
calls = []
real_run = subprocess.run
def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess:
if "install-timers" not in argv:
return real_run(argv, **kwargs)
calls.append(argv)
return subprocess.CompletedProcess(argv, 0)
with unittest.mock.patch.object(campaign.subprocess, "run", fake_run):
self.assertEqual(campaign.main(["install-timer"]), 0)
self.assertEqual(campaign.main(["install-timer", "--enable"]), 0)
self.assertEqual([argv[2:] for argv in calls],
[["install-timers"], ["install-timers", "--enable"]])
self.assertTrue(calls[0][1].endswith("scripts/autorun.py"))
class Registry(unittest.TestCase):
def test_no_profile_takes_a_campaign_file_name(self) -> None:
stems = {p.stem for p in (REPO_ROOT / "emulators").glob("*.yml")}
@@ -942,12 +1072,19 @@ class FakeCorePasses(Workspace):
isolations = {rep["isolation"] for r in records for v in r["variants"] for rep in v["repetitions"]}
self.assertEqual(isolations, {campaign.core_probe.SANDBOX})
kinds = sorted(r["subject"].split("-")[0] for r in records)
self.assertEqual(kinds, ["alias", "bench", "boot", "files"])
boot = next(r for r in records if r["subject"].startswith("boot"))
alias = next(r for r in records if r["subject"].startswith("alias"))
self.assertEqual(kinds, ["alias", "files", "negative", "none"])
by_kind = {r["subject"].split("-")[0]: r for r in records}
self.assertEqual(by_kind["files"]["variants"][0]["verdict"], "boots")
self.assertEqual(by_kind["none"]["variants"][0]["verdict_evidence"]["stages"], ["game"])
self.assertEqual(by_kind["negative"]["check"]["result"]["verdicts"],
{"bios.bin": "boots", "bios.bin-zeros-4": "hang",
"bios.bin-head-2-of-bios.bin": "boots"})
entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0]
self.assertEqual(entry["hle_fallback"], True)
self.assertEqual(entry["evidence"], {"aliases": [alias["id"]], "hle_fallback": [boot["id"]]})
self.assertEqual(entry["validation"], ["size"])
self.assertEqual(entry["evidence"], {
"aliases": [by_kind["alias"]["id"]], "hle_fallback": [by_kind["none"]["id"]],
"validation": [by_kind["none"]["id"], by_kind["negative"]["id"]]})
statuses = {(f["field"], f["status"]) for f in mr.read_campaign(self.root, mr.FINDINGS)}
self.assertEqual(statuses, {("hle_fallback", "applied"), ("required", "open"),
("aliases", "applied")})
@@ -966,9 +1103,23 @@ class FakeCorePasses(Workspace):
with rebuilt.open("ab") as handle:
handle.write(b"\0")
third = self.passing("hle", core=rebuilt)
# BIOS.BIN is declared now, so no alias run follows the boot test.
# BIOS.BIN is declared now, so no alias run follows.
self.assertEqual(len(third.run(set(), None).records), 3)
def test_a_size_check_the_profile_lacks_is_written(self) -> None:
profile = self.root / "emulators" / "fake.yml"
profile.write_text(PROFILE.replace(" validation: [size]\n", ""), encoding="utf-8")
self.passing("hle").run(set(), None)
entry = yaml.safe_load(profile.read_text())["files"][0]
self.assertEqual(entry["validation"], ["size"])
applied = {f["field"] for f in mr.read_campaign(self.root, mr.FINDINGS) if f["status"] == "applied"}
self.assertIn("validation", applied)
def test_bench_runs_only_when_asked(self) -> None:
self.passing("plain", bench=True).run(set(), None)
subjects = [s.record["subject"] for s in mr.read_records(self.root)]
self.assertEqual(sum(s.startswith("bench") for s in subjects), 1)
def test_no_stand_in_settles_required_and_finds_an_alias(self) -> None:
c = self.passing()
c.run(set(), None)
@@ -987,9 +1138,59 @@ class FakeCorePasses(Workspace):
self.assertEqual(left.stdout, b"")
checks = c.state["checks"]["fake"]
self.assertTrue(checks)
self.assertEqual({entry["status"] for entry in checks.values()}, {"failed"})
self.assertEqual({entry["reason"] for entry in checks.values()}, {"timeout"})
self.assertEqual(mr.read_records(self.root), [])
# A run killed after its timeout is a record whose verdict is a hang.
self.assertEqual({entry["status"] for entry in checks.values()}, {"done"})
records = [s.record for s in mr.read_records(self.root)]
self.assertTrue(records)
self.assertEqual({v["verdict"] for r in records for v in r["variants"]}, {"hang"})
self.assertEqual({rep["verdict_evidence"]["rule"] for r in records for v in r["variants"]
for rep in v["repetitions"]}, {"timeout"})
def test_two_passes_interleave_their_runs(self) -> None:
"""Each core_probe takes the host lock for its own run: a pass never
holds it between runs, so a second pass runs between them."""
import threading
import time
spans: dict[str, list[tuple[float, float]]] = {"a": [], "b": []}
other = Workspace(methodName="setUp")
other.setUp()
self.addCleanup(other.tearDown)
passes = []
for name, workspace in (("a", self), ("b", other)):
c = FakeCorePasses.passing(workspace, "hle") if workspace is self else self.passing_in(workspace)
real = c.host.run
def timed(argv: list[str], *args: object, _name: str = name, _real=real, **kwargs: object):
start = time.monotonic()
try:
return _real(argv, *args, **kwargs)
finally:
spans[_name].append((start, time.monotonic()))
c.host.run = timed
passes.append(c)
threads = [threading.Thread(target=c.run, args=(set(), None)) for c in passes]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
ends = {name: sorted(end for _, end in runs) for name, runs in spans.items()}
self.assertGreater(len(ends["a"]), 2)
self.assertGreater(len(ends["b"]), 2)
# Neither pass finished all its runs before the other started finishing its own.
self.assertLess(ends["b"][0], ends["a"][-1])
self.assertLess(ends["a"][0], ends["b"][-1])
def passing_in(self, workspace: Workspace) -> campaign.Campaign:
workspace.put_game()
host = campaign.Host(loadavg=lambda: (0.0, 0.0, 0.0), free_gb=lambda path: 500.0,
cpu_count=1_000_000)
c = campaign.Campaign(workspace.paths, campaign.Settings(
check_repeat=2, probe_timeout=30, delays=(1,), echo=False), host,
lock_path=workspace.tmp / "auto.lock")
so = self.cores["hle"]
c.unit_cores = lambda *a, **k: [campaign.Core("fake", so, campaign.file_sha256(so))]
return c
def test_missing_content_leaves_the_checks_pending(self) -> None:
c = self.passing(romfetch=str(self.tmp / "no-romfetch"))
tests/test_core_probe.py
+285 -1
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import os
@@ -20,6 +21,7 @@ import tempfile
import unittest
import unittest.mock
from pathlib import Path
from typing import ClassVar
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
@@ -42,6 +44,7 @@ FAKE_CORE = r"""
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
struct retro_system_info { const char *n, *v, *e; bool full, block; };
struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; };
struct retro_system_timing { double fps, rate; };
@@ -78,11 +81,15 @@ bool retro_load_game(const void *g) {
if (env(27, &cb)) {
logger = cb;
cb.log(1, "card %s byte %d\n", f ? "present" : "absent", (int)card);
cb.log(1, "tag %s\n", tag ? tag : "unset"); }
cb.log(1, "tag %s\n", tag ? tag : "unset");
if (card == 0xEE) cb.log(3, "BIOS not found: card.bin\n"); }
return true; }
void retro_unload_game(void) {}
void retro_run(void) {
int16_t s[4] = {1, 2, 3, 4};
const char *end = getenv("CORE_PROBE_TEST_END");
if (end && frames == 1 && !strcmp(end, "crash")) { if (logger.log) logger.log(3, "about to fall\n"); raise(SIGSEGV); }
if (end && frames == 1 && !strcmp(end, "hang")) for (;;) {}
if (frames++ == 0 && logger.log) logger.log(1, "ports %u\n", ports);
input(0, 1, 0, 8);
for (int i = 0; i < 8; i++) pixels[i] = card;
@@ -177,6 +184,187 @@ class ParsingTests(unittest.TestCase):
self.assertEqual([p.name for p in core_probe.cue_tracks(cue)], ["a b.bin", "c.bin"])
def frames(*pairs: tuple[int, str]) -> list[list]:
return [[frame, digest] for frame, digest in pairs]
class OracleTests(unittest.TestCase):
"""judge() turns a repetition into a verdict from positive evidence only."""
TITLE, LOGO, ERROR, BLACK, OTHER = ("1" * 40, "2" * 40, "3" * 40, "4" * 40, "5" * 40)
ORACLE: ClassVar[dict] = {"references": [{"sha1": "2" * 40, "stage": "bios", "shows": "the console logo"},
{"sha1": "1" * 40, "stage": "game", "shows": "the title"}],
"error_patterns": [r"BIOS not found"],
"error_screens": [{"sha1": "3" * 40, "shows": "the core's missing-BIOS message"}]}
def rep(self, hashes: list, logs: list[str] = (), uniform: list[str] = (), loaded: bool = True,
**extra: object) -> dict:
return {"loaded": loaded, "library_name": "Fake", "frames": 100, "warmup": 0,
"logs": [{"frame": 0, "level": "error", "text": text} for text in logs],
"observables": {"frame_hashes": hashes, "uniform_frames": list(uniform)}, **extra}
def test_a_reference_frame_is_a_boot_with_its_stages(self) -> None:
verdict, evidence = core_probe.judge(
self.rep(frames((0, self.BLACK), (10, self.LOGO), (20, self.TITLE)),
logs=["BIOS not found"]), self.ORACLE)
self.assertEqual(verdict, "boots")
self.assertEqual((evidence["rule"], evidence["frame"], evidence["stages"]),
("reference_frame", 20, ["bios", "game"]))
def test_a_bios_load_line_stands_for_hidden_bios_screens(self) -> None:
oracle = {**self.ORACLE, "bios_patterns": [r"^found US BIOS file"]}
verdict, evidence = core_probe.judge(
self.rep(frames((0, self.TITLE)), logs=["found US BIOS file, crc32 1: x"]), oracle)
self.assertEqual((verdict, evidence["stages"]), ("boots", ["bios", "game"]))
self.assertEqual(core_probe.judge(self.rep(frames((0, self.TITLE))), oracle)[1]["stages"], ["game"])
def test_the_bios_screens_alone_are_not_a_boot(self) -> None:
# A console that shows its logo and then refuses the disc never reached the game.
verdict, evidence = core_probe.judge(
self.rep(frames((0, self.LOGO), (10, self.BLACK), (20, self.BLACK)), uniform=[self.BLACK]),
self.ORACLE)
self.assertEqual((verdict, evidence["rule"], evidence["stages"]), ("hang", "uniform_tail", ["bios"]))
self.assertIn("no screen of the game", evidence["detail"])
def test_an_error_pattern_or_screen_without_a_reference_is_a_bios_error(self) -> None:
verdict, evidence = core_probe.judge(
self.rep(frames((0, self.OTHER)), logs=["x", "BIOS not found: scph5501.bin"]), self.ORACLE)
self.assertEqual((verdict, evidence["rule"], evidence["log"]),
("bios_error", "error_pattern", "BIOS not found: scph5501.bin"))
verdict, evidence = core_probe.judge(self.rep(frames((0, self.ERROR))), self.ORACLE)
self.assertEqual((verdict, evidence["rule"], evidence["frame"]), ("bios_error", "error_screen", 0))
self.assertEqual(core_probe.judge(self.rep([], logs=["BIOS not found"], loaded=False),
self.ORACLE)[0], "bios_error")
def test_process_status_decides_crash_and_hang_first(self) -> None:
crashed = self.rep(frames((0, self.TITLE)), exit={"status": -11, "signal": "SIGSEGV"})
self.assertEqual(core_probe.judge(crashed, self.ORACLE)[0], "crash")
killed = self.rep([], exit={"status": None, "signal": "SIGKILL", "timeout_s": 5})
self.assertEqual(core_probe.judge(killed, self.ORACLE), (
"hang", {"rule": "timeout", "detail": "killed after 5 s", "exit_status": None,
"signal": "SIGKILL"}))
def test_a_one_colour_second_half_is_a_hang(self) -> None:
run = self.rep(frames((0, self.OTHER), (10, self.OTHER), (20, self.BLACK), (30, self.BLACK)),
uniform=[self.BLACK])
self.assertEqual(core_probe.judge(run, self.ORACLE)[1]["rule"], "uniform_tail")
lit = self.rep(frames((0, self.BLACK), (10, self.BLACK), (20, self.OTHER), (30, self.BLACK)),
uniform=[self.BLACK])
self.assertEqual(core_probe.judge(lit, self.ORACLE)[0], "unknown")
def test_frames_that_differ_are_never_a_boot(self) -> None:
verdict, evidence = core_probe.judge(self.rep(frames((0, self.OTHER), (10, "6" * 40))),
self.ORACLE)
self.assertEqual((verdict, evidence["rule"]), ("unknown", "no_evidence"))
verdict, evidence = core_probe.judge(self.rep(frames((0, self.TITLE))),
{**self.ORACLE, "references": []})
self.assertEqual(verdict, "unknown")
self.assertIn("no reference frame is known", evidence["detail"])
self.assertEqual(core_probe.judge(self.rep([], loaded=False), self.ORACLE)[0], "unknown")
def test_a_variant_needs_every_repetition_to_agree(self) -> None:
boot = {"verdict": "boots", "verdict_evidence": {"rule": "reference_frame", "detail": "d",
"frame": 3, "stages": ["game"]}}
verdict, evidence = core_probe.variant_verdict([boot, boot])
self.assertEqual((verdict, evidence["rule"], evidence["stages"]),
("boots", "repetitions_agree", ["game"]))
hang = {"verdict": "hang", "verdict_evidence": {"rule": "uniform_tail", "detail": "d"}}
verdict, evidence = core_probe.variant_verdict([boot, hang])
self.assertEqual((verdict, evidence["repetitions"]), ("unknown", ["boots", "hang"]))
def test_oracle_entries_are_picked_by_content_and_core(self) -> None:
data = {"error_patterns": [{"pattern": "shared"}],
"cores": {"Fake": {"error_patterns": [{"pattern": "own"}],
"error_screens": [{"sha1": self.ERROR, "shows": "s"}]}},
"content": [{"name": "disc", "sha1": "a" * 40, "references": {
"Fake": {"frames": [{"sha1": self.TITLE, "stage": "game", "shows": "t"}]},
"Other": {"frames": [{"sha1": self.LOGO, "stage": "bios", "shows": "l"}]}}}]}
picked = core_probe.oracle_for(data, "a" * 40, "Fake")
self.assertEqual(picked["references"], [{"sha1": self.TITLE, "stage": "game", "shows": "t"}])
self.assertEqual(picked["error_patterns"], ["shared", "own"])
self.assertEqual(picked["bios_patterns"], [])
self.assertEqual(core_probe.oracle_for(data, "b" * 40, "Fake")["references"], [])
self.assertEqual(core_probe.oracle_for(data, "a" * 40, "Nobody")["error_patterns"], ["shared"])
def test_every_reference_judges_its_own_record_a_boot(self) -> None:
"""Each reference block names the run it was taken from: judged
again with the oracle as it stands, every repetition of that run
boots, so a frame copied wrong or a stage mislabelled fails here."""
data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE)
records = {s.record["id"]: s.record for s in measurement_records.read_records(REPO_ROOT)}
checked = 0
for item in data.get("content") or ():
for library, block in item["references"].items():
record = records.get(block["recorded_from"])
self.assertIsNotNone(record, f"{item['name']} on {library}: {block['recorded_from']}")
self.assertEqual(record["inputs"]["content"]["sha1"], item["sha1"], item["name"])
# A check names the variant that ran the full set, a run has one.
label = block.get("variant")
chosen = next(v for v in record["variants"] if v["label"] == label) if label \
else record["variants"][0]
for rep in chosen["repetitions"]:
oracle = core_probe.oracle_for(data, item["sha1"], rep["library_name"])
verdict, evidence = core_probe.judge(rep, oracle)
self.assertEqual(verdict, "boots", f"{item['name']} on {library}: {evidence}")
checked += 1
self.assertGreaterEqual(checked, 2 * 14)
def test_the_repository_error_patterns_and_screens(self) -> None:
data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE)
saturn = core_probe.oracle_for(data, None, "Beetle Saturn")
for line in ('Cannot open BIOS file "/system/sega_101.bin".',
'BIOS file "/system/sega_101.bin" is of an incorrect size.',
'BIOS file "/system/sega_101.bin" could not be fully read (short or failed read).'):
rep = {"loaded": False, "logs": [{"frame": 0, "level": "error", "text": line}]}
self.assertEqual(core_probe.judge(rep, saturn)[0], "bios_error", line)
menu = {"Beetle PSX": "dfbfc299250c0983f86bbf951efd187441306d92",
"Beetle PSX HW": "dfbfc299250c0983f86bbf951efd187441306d92",
"SwanStation": "167fb7b6527e295b12dca5806a29856f5f31d401",
"pcsx2": "46329c633ea7d0b63349261e4aab169e4f44d3f6"}
for library, sha1 in menu.items():
rep = {"loaded": True, "logs": [], "frames": 100, "warmup": 0,
"observables": {"frame_hashes": [[90, sha1]], "uniform_frames": []}}
verdict, evidence = core_probe.judge(rep, core_probe.oracle_for(data, None, library))
self.assertEqual((verdict, evidence["rule"]), ("bios_error", "error_screen"), library)
# The PlayStation cores fall back to a built-in BIOS: nothing they log is an error.
for library in ("Beetle PSX", "SwanStation", "PCSX-ReARMed", "PCSX1"):
self.assertEqual(core_probe.oracle_for(data, None, library)["error_patterns"], [])
def test_the_repository_oracle_is_well_formed(self) -> None:
import yaml
data = yaml.safe_load(core_probe.DEFAULT_ORACLE.read_text(encoding="utf-8"))
patterns = list(data.get("error_patterns") or ())
for core in (data.get("cores") or {}).values():
patterns += (core.get("error_patterns") or []) + (core.get("bios_patterns") or [])
for screen in core.get("error_screens") or ():
self.assertRegex(screen["sha1"], r"^[0-9a-f]{40}$")
for pattern in patterns:
self.assertTrue(pattern.get("source_ref") or pattern.get("reason"), pattern)
core_probe.re.compile(pattern["pattern"])
for item in data.get("content") or ():
self.assertRegex(item["sha1"], r"^[0-9a-f]{40}$")
for block in item["references"].values():
self.assertRegex(block["recorded_from"], r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
stages = {frame["stage"] for frame in block["frames"]}
self.assertIn("game", stages, item["name"])
for frame in block["frames"]:
self.assertRegex(frame["sha1"], r"^[0-9a-f]{40}$")
self.assertIn(frame["stage"], ("bios", "game"))
self.assertTrue(frame["shows"])
def test_made_files(self) -> None:
zeros = core_probe.Synthetic.parse("@zeros:16")
self.assertEqual((zeros.label("bios.bin"), zeros.source), ("bios.bin-zeros-16", None))
head = core_probe.Synthetic.parse("@head:8:/x/scph5501.bin")
self.assertEqual(head.label("bios.bin"), "bios.bin-head-8-of-scph5501.bin")
for bad in ("@zeros:", "@zeros:8:/x", "@head:8", "@ones:8", "@head:x:/y"):
with self.assertRaises(argparse.ArgumentTypeError):
core_probe.Synthetic.parse(bad)
self.assertEqual(core_probe.exit_description(-11), {"status": -11, "signal": "SIGSEGV"})
self.assertEqual(core_probe.exit_description(139), {"status": 139, "signal": "SIGSEGV"})
self.assertEqual(core_probe.exit_description(1), {"status": 1, "signal": None})
class ResultTests(unittest.TestCase):
@staticmethod
def variant(label: str, value: str, loaded: bool = True) -> dict:
@@ -290,6 +478,102 @@ class FakeCoreTests(unittest.TestCase):
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [])
def oracle_file(self) -> Path:
title = hashlib.sha1(struct.pack("<8H", *[1] * 8)).hexdigest()
content = self.tmp / "game.bin"
content.write_bytes(b"game")
path = self.tmp / "oracle.yml"
path.write_text(
"error_patterns:\n - pattern: 'BIOS not found'\n reason: test\n"
"cores:\n Fake:\n error_patterns: []\n error_screens: []\n"
f"content:\n - name: game\n sha1: {core_probe.file_digest(content)}\n"
" references:\n Fake:\n recorded_from: 2026-10-27-000000000000\n"
f" frames:\n - sha1: {title}\n stage: game\n"
" shows: card one\n", encoding="utf-8")
return path
def test_verdicts_of_a_negative_check(self) -> None:
oracle = self.oracle_file()
error = self.tmp / "error.bin"
error.write_bytes(b"\xee")
code, record = self.probe(["check", "--core", str(self.core), "--frames", "4",
"--frame-hash-every", "1", "--content", str(self.tmp / "game.bin"),
"--oracle", str(oracle), "--vary", "card.bin=",
"--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={error}",
"--vary", "card.bin=@zeros:1",
"--vary", f"card.bin=@head:0:{self.cards[0]}",
"--observable", "frame_hashes", "--claim", "c", "--method", "m",
"--no-trace", "--repeat", "2"])
self.assertEqual(code, 0)
verdicts = record["check"]["result"]["verdicts"]
self.assertEqual(verdicts, {"card.bin-absent": "hang", "card0.bin": "boots",
"error.bin": "bios_error", "card.bin-zeros-1": "hang",
"card.bin-head-0-of-card0.bin": "hang"})
boot = record["variants"][1]
self.assertEqual((boot["verdict_evidence"]["rule"], boot["verdict_evidence"]["stages"]),
("repetitions_agree", ["game"]))
self.assertEqual(boot["repetitions"][0]["verdict_evidence"]["rule"], "reference_frame")
self.assertEqual(record["variants"][3]["system_files"],
{"card.bin": hashlib.sha1(b"\x00").hexdigest()})
self.assertEqual(record["inputs"]["oracle"]["error_patterns"], ["BIOS not found"])
self.assertEqual(record["record_version"], 4)
self.assert_schema(record)
def test_a_crash_and_a_hang_are_verdicts(self) -> None:
oracle = self.oracle_file()
code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",
"--frame-hash-every", "1", "--oracle", str(oracle),
"--env", "CORE_PROBE_TEST_END=crash"])
self.assertEqual(code, 0)
crashed = record["variants"][0]
self.assertEqual((crashed["verdict"], crashed["repetitions"][0]["exit"]["signal"]),
("crash", "SIGSEGV"))
self.assertEqual(crashed["repetitions"][0]["logs"][-1]["text"], "about to fall")
self.assertEqual(crashed["repetitions"][0]["observables"]["frame_hashes"][0][0], 0)
self.assert_schema(record)
code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",
"--frame-hash-every", "1", "--oracle", str(oracle), "--timeout", "3",
"--env", "CORE_PROBE_TEST_END=hang", "--no-trace"])
self.assertEqual(code, 0)
self.assertEqual((record["variants"][0]["verdict"],
record["variants"][0]["repetitions"][0]["verdict_evidence"]["rule"]),
("hang", "timeout"))
def assert_schema(self, record: dict) -> None:
try:
from jsonschema import Draft202012Validator
except ImportError:
return
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
sealed = measurement_records.seal({**record, "emulator": "fake", "subject": "s"})
self.assertEqual([e.message for e in Draft202012Validator(schema).iter_errors(sealed)], [])
def test_a_run_waits_for_the_host_lock_and_frees_it(self) -> None:
import fcntl
import time
lock = self.tmp / "host.lock"
argv = [sys.executable, str(REPO_ROOT / "scripts" / "core_probe.py"), "run", "--core",
str(self.core), "--frames", "1", "--no-trace", "--work-dir", str(self.tmp / "work")]
env = {**os.environ, "RETROBIOS_PROBE_LOCK": str(lock)}
with lock.open("a") as held:
fcntl.flock(held, fcntl.LOCK_EX)
waiting = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
text=True)
time.sleep(1.5)
self.assertIsNone(waiting.poll(), "the run went ahead while another held the lock")
# A caller that holds the lock for this run says so.
held_run = subprocess.run(argv, env={**env, "RETROBIOS_PROBE_LOCK_HELD": "1"},
capture_output=True, text=True, timeout=60, check=False)
self.assertEqual(held_run.returncode, 0, held_run.stderr)
fcntl.flock(held, fcntl.LOCK_UN)
_, err = waiting.communicate(timeout=60)
self.assertEqual(waiting.returncode, 0, err)
self.assertIn("another measurement is running", err)
# Released at the end of the run: the lock is free again.
with lock.open("a") as again:
fcntl.flock(again, fcntl.LOCK_EX | fcntl.LOCK_NB)
def test_second_run_is_appended_beside_the_first(self) -> None:
record_dir = checkout(self.tmp / "appended")
argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000",
tests/test_measurements.py
+185 -23
View File
@@ -43,16 +43,21 @@ def repetition(value: str, frames: int = 3) -> dict:
"last_frame_size": [4, 2], "tracing": "strace", "input_queries": 3, "input_pressed": 1,
"files_opened": [{"path": "card.bin", "sha1_after": SHA1, "sha1_staged": SHA1}],
"files_probed_absent": [],
"verdict": "boots",
"verdict_evidence": {"rule": "reference_frame", "detail": "reached the title", "frame": 150,
"sha1": value, "stages": ["game"]},
}
def full_record(emulator: str, subject: str, when: str, kind: str = "check",
values: tuple[str, ...] = ("1" * 40, "2" * 40), observable: str = "video_sha1") -> dict:
variants = [{"label": f"v{i}", "system_files": {"card.bin": SHA1},
"repetitions": [repetition(v), repetition(v)], "flags": []}
"repetitions": [repetition(v), repetition(v)], "flags": [], "verdict": "boots",
"verdict_evidence": {"rule": "repetitions_agree", "detail": "both boot",
"repetitions": ["boots", "boots"]}}
for i, v in enumerate(values)]
record = {
"record_version": mr.RECORD_VERSION, "emulator": emulator, "subject": subject,
"record_version": 4, "emulator": emulator, "subject": subject,
"supersedes": None, "kind": kind, "recorded_at": when,
"core": {"file": "fake_libretro.so", "sha256": SHA256, "repo": "https://example.org/fake",
"commit": "c" * 40, "build_command": "make", "compiler": ["GCC 13"],
@@ -67,7 +72,10 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
"options": {"fake_option": "a"}, "env": {}, "frames": 3, "warmup": 0,
"repetitions": 2 if kind == "check" else 5, "presses": [],
"observables": ["frame_sha1", "video_sha1", "audio_sha1", "state_sha1", "frame_hashes"],
"frame_hash_every": 150, "joypad_ports": 2},
"frame_hash_every": 150, "joypad_ports": 2,
"oracle": {"file": "tests/campaign/oracle.yml", "sha256": SHA256,
"references": [{"sha1": values[0], "stage": "game", "shows": "title"}],
"error_patterns": ["BIOS not found"], "error_screens": []}},
"variants": variants,
"flags": [],
}
@@ -81,7 +89,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
"observable": observable, "variable": "card.bin",
"result": {"status": "differs" if len(groups) > 1 else "same",
"statement": f"{observable} takes {len(groups)} values",
"groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()]},
"groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()],
"verdicts": {v["label"]: v["verdict"] for v in variants}},
}
else:
record["variants"] = record["variants"][:1]
@@ -211,7 +220,8 @@ class ReadAndIndex(Workspace):
keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind",
"claim", "method", "observable", "variable", "status", "statement", "content",
"variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id",
"runs", "record_url", "sqlite"}
"runs", "record_url", "sqlite", "verdicts", "title", "host_short", "groups",
"proves", "timing", "setup"}
for item in index["items"]:
self.assertEqual(set(item), keys)
alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"}
@@ -225,6 +235,23 @@ class ReadAndIndex(Workspace):
written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text())
self.assertEqual(written, index)
def test_a_frame_check_shows_its_verdicts_never_frames_that_differ(self) -> None:
record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", observable="frame_hashes")
record["variants"][1]["verdict"] = "hang"
self.assertEqual(sm.result_of(record), ("mixed", "v0: boots. v1: hang"))
self.assertEqual([v["verdict"] for v in sm.verdicts_of(record)], ["boots", "hang"])
old = {**record, "record_version": 3,
"variants": [{k: v for k, v in variant.items() if not k.startswith("verdict")}
for variant in record["variants"]]}
status, statement = sm.result_of(old)
self.assertEqual(status, "no verdict")
self.assertNotIn("differ", statement)
# A check on another observable keeps its comparison.
self.assertEqual(sm.result_of(full_record("alpha", "c", "2026-10-06T10:00:00+00:00"))[0], "differs")
groups = sm.variant_groups(record)
self.assertEqual([(g["variant"], g["result"], g["group"]) for g in groups],
[("v0", "works", 0), ("v1", "hang", 1)])
def test_links_name_the_last_commit_that_touched_the_records(self) -> None:
first = self.repository()
(self.tree / "other.txt").write_text("x")
@@ -304,6 +331,26 @@ class ReadAndIndex(Workspace):
self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
self.assertIsNotNone(Draft202012Validator)
def test_a_version_4_record_needs_its_verdicts(self) -> None:
try:
from jsonschema import Draft202012Validator # noqa: F401
except ImportError:
self.skipTest("jsonschema is not installed")
import validate_schemas # noqa: PLC0415
lines = self.tree / mr.records_path("beta")
record = json.loads(lines.read_text())
del record["variants"][0]["verdict"]
del record["variants"][0]["repetitions"][0]["verdict_evidence"]
lines.write_text(mr.line_text(record) + "\n")
errors = validate_schemas._measurement_errors(self.tree)
self.assertTrue(any("'verdict' is a required property" in e for e in errors), errors)
self.assertTrue(any("'verdict_evidence' is a required property" in e for e in errors), errors)
old = {**record, "record_version": 3}
lines.write_text(mr.line_text(old) + "\n")
self.assertFalse(any("required property" in e
for e in validate_schemas._measurement_errors(self.tree)))
class AppendOnly(Workspace):
def setUp(self) -> None:
@@ -383,26 +430,141 @@ class SitePages(unittest.TestCase):
self.assertEqual(sm.tested_section("beetle_saturn", []), [])
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM]))
self.assertTrue(text.startswith("## Tested\n"))
for expected in ('<span class="rb-badge rb-badge-muted">same</span>',
"**Method:** Twice per card. Every input fixed.",
"Ran on Test CPU, 12 CPUs, 2026-10-27.", "Played: disc.",
self.ITEM["record_url"], "1 flag raised", "2 runs",
"../downloads/measurements/beetle_saturn.sqlite", "../wiki/measurements.md",
"&lt;image>", '<table class="rb-entries"'):
for expected in ("same with every file", "<strong>Method:</strong> Twice per card. Every input fixed.",
"<strong>Ran on:</strong> Test CPU, 12 CPUs", self.ITEM["record_url"],
"1 flag raised", "2 runs", "downloads/measurements/beetle_saturn.sqlite",
"wiki/measurements/", "&lt;image&gt;", "1 test · last run 2026-10-27",
'<p class="rb-tested-file"><code>mpeg_card.bin</code></p>',
'class="rb-tested-legend"', "No file staged."):
self.assertIn(expected, text)
self.assertNotIn('<th scope="col">Game</th>', text)
self.assertNotIn("confirmed", text)
self.assertNotIn(" open", text)
def test_tested_block_says_shared_context_once(self) -> None:
other = {**self.ITEM, "subject": "other", "content": "cart", "status": "inconclusive",
"host": "Test CPU, 12 CPUs, load 1.5 before, 1.6 after",
"core": {**self.ITEM["core"], "commit": "f" * 40}}
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM, other]))
self.assertEqual(text.count("Test CPU, 12 CPUs"), 1)
self.assertIn('<th scope="col">Game</th>', text)
self.assertNotIn("Played:", text)
self.assertIn(f"Core build: Beetle Saturn v1.32.1 at `{'f' * 12}`.", text)
self.assertIn("Load 1.5 before, 1.6 after.", text)
self.assertIn('<span class="rb-badge rb-badge-warning">inconclusive</span>', text)
def item(self, **values: object) -> dict:
return {**self.ITEM, "host_short": "Test CPU, Linux", **values}
def test_summary_counts(self) -> None:
items = [self.item(proves=["hle_fallback of a.bin"], open_findings=2),
self.item(id="b", subject="b", date="2026-10-28T00:00:00+00:00", open_findings=2),
self.item(id="c", kind="bench", status="measured", subject="c",
timing={"ms_per_frame": 1.5, "fps": 660.0, "stdev": 0.01})]
self.assertEqual(sm.tested_counts(items), {"tests": 3, "confirmed": 1, "open": 2, "benchmarks": 1,
"last": "2026-10-28"})
text = "\n".join(sm.tested_section("beetle_saturn", items))
self.assertIn("3 tests · 1 confirmed · 2 open · last run 2026-10-28 on Test CPU, Linux.", text)
self.assertIn("<td>1.500</td><td>660.0</td>", text)
def test_short_titles(self) -> None:
cases = [
({"subject": "mpeg-card-lunar-cr2-build", "variable": "mpeg_card.bin",
"content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"},
"mpeg_card.bin, Lunar (CR2 build)"),
({"subject": "negative-scph5501-bin-mednafen-psx-bust-a-move-2-arcade-edition-usa",
"variable": "scph5501.bin", "content": "Bust-A-Move 2 - Arcade Edition (USA)",
"core": {"file": "mednafen_psx_libretro.so"}},
"scph5501.bin, Bust-A-Move 2 (wrong files)"),
({"subject": "issue85-lunar-opening-movie", "variable": None,
"content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"},
"Lunar (issue 85 opening movie)"),
({"subject": "with-command-auth-device-and-command-get-9a211715", "variable": "mpeg_card.bin",
"content": "Redump: Lunar"},
"mpeg_card.bin, Lunar (with command auth device and get)"),
]
for item, title in cases:
self.assertEqual(sm.short_title(item), title)
self.assertEqual(sm.short_host({"cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K",
"os": "Linux-6.18.54.1-microsoft-standard-WSL2-x86_64"}),
"Intel Core i9-13900K, Linux (WSL2)")
def test_rows_of_one_file_never_read_alike(self) -> None:
def item(rid: str, build: str, files: int, observable: str, clock: bool = True) -> dict:
record = {"core": {"build_command": build}, "subject": "mpeg-card-lunar", "kind": "check",
"check": {"observable": observable}, "variants": [{}] * files,
"inputs": {"options": {"beetle_saturn_mpeg_card": "enabled"},
"env": {"FAKETIME": "x"} if clock else {}}}
return self.item(id=rid, content="Redump: Lunar - Silver Star Story", setup=sm.setup_of(record))
items = [item("a", "make -j8", 11, "video_sha1"),
item("b", "make -j8 after editing cdb.c", 11, "video_sha1"),
item("c", "make -j10, with the CR2 patch to cdb.c", 3, "state:MPEG.AuthState"),
item("d", "make -j8", 11, "video_sha1", clock=False)]
titles = sm.distinct_titles(items)
self.assertEqual(titles, {
"a": "Lunar",
"b": "Lunar, patched build",
"c": "Lunar, CR2 patch, 3 files, MPEG.AuthState",
"d": "Lunar, running clock"})
self.assertEqual(len(set(titles.values())), len(items))
self.assertNotIn("...", " ".join(titles.values()))
# Alone in its file, a test is named by its game.
self.assertEqual(sm.distinct_titles(items[:1]), {"a": "Lunar"})
def test_run_output_is_text_never_markup(self) -> None:
nasty = "`code` {expr} <script>alert(1)</script> [x](javascript:alert(1)) \"q\" 'q' **b**"
item = self.item(
claim=nasty, method=nasty, statement=nasty, host=nasty, host_short=nasty, content=nasty,
variable=f"{nasty}.bin", subject="s", title=nasty, proves=[nasty], runs=2, flags=1,
core={"library": nasty, "commit": "{`<x>`}", "file": f"{nasty}_libretro.so"},
groups=[{"variant": nasty, "group": 0, "result": nasty, "absent": False}],
record_url="javascript:alert(1)")
bench = self.item(id="b", kind="bench", status="measured", subject="bench", title=nasty,
content=nasty, record_url="java\nscript:x",
timing={"ms_per_frame": 1.0, "fps": 1.0, "stdev": 0.0})
earlier = self.item(id="e", status="no verdict", subject="e", content=nasty, claim=nasty)
text = "\n".join(sm.tested_section("alpha", [item, bench, earlier]))
self.assertNotIn("<script", text)
self.assertNotRegex(text, r'href="(?!https?://|/)')
self.assertNotIn("javascript:alert(1)\"", text)
self.assertNotRegex(text, r"[{}]")
self.assertNotIn("<strong>b</strong>", text)
self.assertNotIn("<code>code</code>", text)
self.assertNotRegex(text, r'="[^"]*"q"')
self.assertIn("&lt;script&gt;", text)
self.assertIn("&#123;expr&#125;", text)
def test_no_empty_sentence(self) -> None:
text = "\n".join(sm.tested_section("beetle_saturn", [self.item(claim=None, method=None, statement="",
kind="run", status="ran")]))
self.assertNotRegex(text, r"</strong>\s*\.|:\s*\.\s|`ran`\. \.")
self.assertNotIn("Claim:", text)
self.assertNotIn("Method:", text)
def test_dots_follow_the_result_groups(self) -> None:
record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", values=("1" * 40, "2" * 40, "1" * 40))
record["check"]["variable"] = "card.bin"
record["variants"][2]["system_files"] = {"card.bin": None}
groups = sm.variant_groups(record)
self.assertEqual([g["group"] for g in groups],
[i for i, group in enumerate(record["check"]["result"]["groups"])
for _ in group["variants"]][:1] + [1, 0])
self.assertEqual([g["absent"] for g in groups], [False, False, True])
text = "\n".join(sm.tested_section("alpha", [self.item(groups=groups)]))
strip = text.split('class="rb-dots"', 1)[1].split("</span></span>", 1)[0]
self.assertEqual(re.findall(r'rb-dot rb-dot-(\d)( rb-dot-empty)?"', strip),
[("0", ""), ("1", ""), ("0", " rb-dot-empty")])
self.assertIn('aria-label="3 variants. v0: result A. v1: result B. v2: result A"', text)
def test_the_details_hold_the_method_and_the_host(self) -> None:
text = "\n".join(sm.tested_section("beetle_saturn", [self.item(runs=1)]))
details = text.split('<div class="rb-tested-more">', 1)[1].split("</div>", 1)[0]
self.assertIn("Twice per card", details)
self.assertIn("<strong>Ran on:</strong> Test CPU, Linux.", details)
self.assertIn(f'<a href="{self.ITEM["record_url"]}">Full record</a>.', details)
self.assertNotIn("1 run", details)
def test_the_beetle_saturn_block_stays_short(self) -> None:
stored = mr.read_records(REPO_ROOT)
profiles = sm.load_profiles(REPO_ROOT / "emulators")
index = sm.build_index(stored, {"repo": REPO_URL, "commit": COMMIT}, profiles)
items = sm.by_emulator(index).get("beetle_saturn")
if not items:
self.skipTest("no beetle_saturn record")
block = "\n".join(sm.tested_section("beetle_saturn", items))
# What a reader sees before opening anything: the closed details hide their body.
shown = re.sub(r'<div class="rb-tested-more">.*?</div>', " ", block, flags=re.S)
shown = re.sub(r'<details class="rb-tested-earlier">.*?</details>\s*</details>', " ", shown,
flags=re.S)
shown = re.sub(r"<[^>]+>", " ", shown)
self.assertLess(len(re.findall(r"[A-Za-z0-9][\w.'-]*", shown)), 120, shown)
def test_hosts_table_on_the_method_page(self) -> None:
page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n",
tests/test_region_check.py
+288
View File
@@ -0,0 +1,288 @@
"""region_check: the variants a region game runs under and what their
records settle."""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
import region_check as rc # noqa: E402
PROFILE = {
"systems": ["sony-playstation"],
"files": [
{"name": "scph5500.bin", "region": ["japan"]},
{"name": "scph5501.bin", "region": ["north-america", "latin-america"]},
{"name": "scph5502.bin", "region": ["europe", "oceania"]},
{"name": "openbios.bin"},
{"name": "ps1_rom.bin", "region": ["world"]},
{"name": "standalone.bin", "region": ["japan"], "mode": "standalone"},
{"name": "other.bin", "region": ["japan"], "system": "sony-playstation-2"},
],
}
def record(rid: str, staged: list[str], opened: list[str], hashes: list, verdict: str = "unknown",
loaded: bool = True) -> dict:
return {"id": rid, "variants": [{
"label": "default", "verdict": verdict, "system_files": {name: "0" * 40 for name in staged},
"repetitions": [{"loaded": loaded, "files_opened": [{"path": p} for p in opened],
"observables": {"frame_hashes": hashes}}]}]}
class Entries(unittest.TestCase):
def test_only_regional_bios_of_the_system_in_the_libretro_build(self) -> None:
names = [e["name"] for e in rc.region_entries(PROFILE, "sony-playstation")]
self.assertEqual(names, ["scph5500.bin", "scph5501.bin", "scph5502.bin"])
def test_variants_split_by_the_game_region(self) -> None:
entries = rc.region_entries(PROFILE, "sony-playstation")
plan = rc.plan_variants(entries, "japan", lambda e: Path("/bios") / e["name"])
self.assertEqual(sorted(plan["full"]), ["scph5500.bin", "scph5501.bin", "scph5502.bin"])
self.assertEqual(sorted(plan["own"]), ["scph5500.bin"])
self.assertEqual(sorted(plan["others"]), ["scph5501.bin", "scph5502.bin"])
def test_a_file_the_collection_lacks_is_in_no_variant(self) -> None:
entries = rc.region_entries(PROFILE, "sony-playstation")
plan = rc.plan_variants(entries, "europe",
lambda e: None if e["name"] == "scph5502.bin" else Path(e["name"]))
self.assertEqual(plan["own"], {})
self.assertNotIn("scph5502.bin", plan["full"])
def test_subject_names_core_game_and_variant(self) -> None:
slug = rc.subject("mednafen_psx", "Puzzle Bobble 2 (Japan)", "own")
self.assertEqual(slug, "region-mednafen_psx-puzzle-bobble-2-japan-own")
self.assertRegex(slug, rc.SUBJECT)
class BusyHost(unittest.TestCase):
def _run(self, codes: list[int], max_wait: float) -> tuple[int, int]:
import subprocess
now = [0.0]
calls = []
def runner(argv, **_kwargs):
calls.append(argv)
return subprocess.CompletedProcess(argv, codes[min(len(calls), len(codes)) - 1], "", "")
def pause(seconds: float) -> None:
now[0] += seconds
proc = rc.probe_when_idle(["probe"], max_wait, clock=lambda: now[0], pause=pause, runner=runner)
return proc.returncode, len(calls)
def test_a_refused_run_is_tried_again_once_the_host_settles(self) -> None:
self.assertEqual(self._run([rc.BUSY, rc.BUSY, 0], 600), (0, 3))
def test_it_gives_up_after_the_wait(self) -> None:
code, calls = self._run([rc.BUSY], 60)
self.assertEqual(code, rc.BUSY)
self.assertEqual(calls, 60 // rc.POLL_S + 1)
def test_another_failure_is_not_retried(self) -> None:
self.assertEqual(self._run([1, 0], 600), (1, 1))
class Outcome(unittest.TestCase):
ENTRIES = rc.region_entries(PROFILE, "sony-playstation")
ALL = ["scph5500.bin", "scph5501.bin", "scph5502.bin"]
def test_locked_bios_by_verdict(self) -> None:
records = {
"full": record("f", self.ALL, ["scph5500.bin"], [[0, "a"]], "boots"),
"own": record("o", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"),
"others": record("x", self.ALL[1:], [], [[0, "b"]], "bios_error", loaded=False),
"pack": record("p", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"),
}
result = rc.outcome(records, self.ENTRIES, "japan")
self.assertEqual(result["selected"], ["scph5500.bin"])
self.assertEqual(result["claim"], "confirmed")
self.assertEqual((result["own_boots"], result["others_boot"], result["pack_boots"]),
(True, False, True))
self.assertEqual(result["judged_by"], "verdict")
self.assertEqual(rc.sentence("japan", result),
"The Japanese BIOS boots the Japanese game, the other regions' BIOS do not.")
def test_interim_rule_counts_the_same_frames_as_a_boot_and_nothing_else(self) -> None:
records = {
"full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"], [10, "b"]]),
"own": record("o", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"], [10, "b"]]),
"others": record("x", ["scph5500.bin", "scph5502.bin"], ["scph5500.bin"], [[0, "a"], [10, "c"]]),
}
result = rc.outcome(records, self.ENTRIES, "north-america")
self.assertEqual(result["judged_by"], "full_run_frames")
self.assertTrue(result["own_boots"])
# Other frames are no verdict: a US disc reaches its game on the
# Japanese BIOS through another licence screen.
self.assertIsNone(result["others_boot"])
self.assertIsNone(result["pack_boots"])
self.assertEqual(rc.sentence("north-america", result),
"The US BIOS boots the US game. The other regions' BIOS have no verdict yet.")
def test_no_interim_answer_when_the_full_run_fails(self) -> None:
records = {"full": record("f", self.ALL, [], [[0, "a"]], "bios_error"),
"own": record("o", ["scph5502.bin"], [], [[0, "a"]])}
result = rc.outcome(records, self.ENTRIES, "europe")
self.assertIsNone(result["own_boots"])
self.assertEqual(result["claim"], "unread")
def test_a_core_reading_another_region_contradicts_the_claim(self) -> None:
records = {"full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"]], "boots")}
self.assertEqual(rc.outcome(records, self.ENTRIES, "japan")["claim"], "contradicted")
def test_a_core_opening_every_region_file_settles_nothing(self) -> None:
"""PCSX-ReARMed opens the three regional files whatever the disc."""
records = {"full": record("f", self.ALL, self.ALL, [[0, "a"]], "boots")}
self.assertEqual(rc.outcome(records, self.ENTRIES, "japan")["claim"], "ambiguous")
def test_region_free_bios(self) -> None:
records = {"full": record("f", self.ALL, ["scph5501.bin"], [[0, "a"]], "boots"),
"own": record("o", ["scph5502.bin"], [], [[0, "a"]], "boots"),
"others": record("x", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"]], "boots")}
result = rc.outcome(records, self.ENTRIES, "europe")
self.assertEqual(rc.sentence("europe", result),
"The European BIOS boots the European game, and so do the other regions' BIOS.")
def test_no_own_run(self) -> None:
self.assertEqual(rc.sentence("japan", rc.outcome({}, self.ENTRIES, "japan")),
"The Japanese game has no run with its own BIOS yet.")
class ContentList(unittest.TestCase):
def test_every_region_system_has_a_game_per_region_or_says_why(self) -> None:
"""A system whose BIOS a profile ties to JP, US and EU has a game for
each of the three, or the content list says why it has none."""
from common import load_emulator_profiles
import yaml
data = yaml.safe_load(rc.CONTENT_FILE.read_text(encoding="utf-8"))
content = rc.load_content()
uncovered = data.get("uncovered") or {}
profiles = load_emulator_profiles(str(ROOT / "emulators"))
regional: set[str] = set()
for profile in profiles.values():
for system in profile.get("systems") or ():
regions = {r for e in rc.region_entries(profile, system) for r in rc.regions_of(e)}
if {"japan", "north-america", "europe"} <= regions:
regional.add(system)
for system in sorted(regional):
if system in uncovered:
continue
held = {item.get("region") for item in content.get(system, [])}
missing = {"japan", "north-america", "europe"} - held
# One cartridge sold in two territories counts for both.
missing -= {r for item in content.get(system, []) for r in item.get("also_regions") or ()}
self.assertFalse(missing, f"{system}: no game for {sorted(missing)}")
class RequiredFromTheRegionRuns(unittest.TestCase):
def test_beetle_psx_boots_each_region_without_its_bios(self) -> None:
"""libretro.c maps its built-in OpenBIOS when the region's file is
absent: the evidence of required: false is a record where the file
was not staged and the oracle judged the run a boot."""
import json
import yaml
profile = yaml.safe_load((ROOT / "emulators" / "beetle_psx.yml").read_text(encoding="utf-8"))
records = {}
for line in (ROOT / "measurements" / "beetle_psx.jsonl").read_text(encoding="utf-8").splitlines():
record = json.loads(line)
records[record["id"]] = record
regional = [e for e in profile["files"] if e["name"] in ("scph5500.bin", "scph5501.bin", "scph5502.bin")]
self.assertEqual(len(regional), 3)
for entry in regional:
with self.subTest(entry=entry["name"]):
self.assertIs(entry["required"], False)
self.assertIn("3425-3511", entry["source_ref"])
ids = entry["evidence"]["required"]
self.assertTrue(ids)
for rid in ids:
record = records[rid]
self.assertEqual(record["record_version"], 4)
variant = record["variants"][0]
self.assertNotIn(entry["name"], variant["system_files"])
self.assertEqual(variant["verdict"], "boots")
self.assertIn("game", variant["verdict_evidence"].get("stages", []))
class SitePages(unittest.TestCase):
def test_region_section_links_every_record(self) -> None:
import sitemeasurements as sm
records = {"full": record("f", Outcome.ALL, ["scph5500.bin"], [[0, "a"]], "boots"),
"own": record("o", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"),
"others": record("x", Outcome.ALL[1:], [], [[0, "b"]], "bios_error")}
result = rc.outcome(records, Outcome.ENTRIES, "japan")
result.update(core="mednafen_psx", game="Puzzle Bobble 2 (Japan)",
sentence=rc.sentence("japan", result))
for label, item in result["variants"].items():
item["url"] = f"https://example.org/{label}"
text = "\n".join(sm.region_section([result]))
self.assertIn("- **mednafen&#95;psx**: Each region's game boots on its own BIOS only.", text)
self.assertIn("The Japanese BIOS boots the Japanese game, the other regions&#x27; BIOS do not.", text)
# The per-game lines and their records wait in a closed fold.
self.assertIn('<details class="rb-region-runs">', text)
self.assertIn('<a href="https://example.org/own">its own region&#x27;s BIOS only</a>', text)
self.assertIn("<code>scph5500.bin</code>", text)
self.assertNotIn("reference frames are recorded", text)
self.assertEqual(sm.region_section([]), [])
def test_the_emulator_page_folds_region_runs_into_one_line(self) -> None:
import measurement_records as mr
import sitemeasurements as sm
stored = mr.read_records(ROOT)
index = sm.build_index(stored, {"repo": sm.REPO, "commit": "0" * 40}, sm.load_profiles(ROOT / "emulators"))
items = sm.by_emulator(index).get("beetle_psx") or []
regional = [i for i in items if sm.REGION_SUBJECT.match(i["subject"])]
if not regional:
self.skipTest("no region record")
text = "\n".join(sm.tested_section("beetle_psx", items))
self.assertIn(f"Region check: {len(regional)} runs of 3 games, read on the system page.", text)
for item in regional:
self.assertNotIn(item["record_url"], text)
def test_coverage_names_tested_and_uncovered_systems(self) -> None:
import sitemeasurements as sm
cov = {"systems": [{"system": "sony-playstation", "tested": True, "games": [
{"name": "A (USA)", "region": "north-america", "emulators": ["beetle_psx"]}]},
{"system": "sega-32x", "tested": False, "games": [
{"name": "B (World)", "region": None, "emulators": []}]}],
"uncovered": [{"system": "msx", "want": "Gradius", "reason": "No MSX catalog.",
"romfetch": {"exit": 1, "code": "unknown_system"}}]}
text = "\n".join(sm.coverage_section(cov, {"sony-playstation": "PlayStation"}))
self.assertIn("1 of 2 listed systems have a recorded run, and 1 system has no test game", text)
self.assertIn("| PlayStation | A (USA) | beetle&#95;psx |", text)
self.assertIn("| sega-32x | B (World) | not yet: no run recorded |", text)
self.assertIn("exit 1, <code>unknown&#95;system</code>", text)
def test_one_catalog_name_on_two_systems_counts_only_where_it_ran(self) -> None:
import measurement_records as mr
import sitemeasurements as sm
label = "Bust-A-Move 2 - Arcade Edition (USA)"
stored = [mr.Stored("beetle_psx", 1, {"inputs": {"content": {"label": label}}})]
cov = sm.coverage(ROOT, stored, {"beetle_psx": {"systems": ["sony-playstation"]}})
by_system = {item["system"]: item for item in cov["systems"]}
self.assertTrue(by_system["sony-playstation"]["tested"])
self.assertFalse(by_system["sega-saturn"]["tested"])
def test_coverage_reads_the_real_content_list(self) -> None:
import sitemeasurements as sm
cov = sm.coverage(ROOT, [])
systems = {item["system"] for item in cov["systems"]}
self.assertIn("sony-playstation", systems)
self.assertTrue(all(item["romfetch"].get("exit") for item in cov["uncovered"]))
if __name__ == "__main__":
unittest.main()
tests/test_standalone_probe.py
+201
View File
@@ -0,0 +1,201 @@
"""standalone_probe: the platform's asset choice, the AppImage offset, the
framebuffer reader, the settings it writes and the oracle it judges with."""
from __future__ import annotations
import struct
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
import core_probe as cp # noqa: E402
import standalone_probe as sp # noqa: E402
def xwd(width: int, height: int, pixels: bytes, ncolors: int = 0, pad: int = 0) -> bytes:
"""An XWD file as Xvfb keeps its screen: version 7, 32 bpp, LSB first."""
name = b"Xvfb main window\0"
header_size = 100 + len(name)
bytes_per_line = width * 4 + pad
fields = [header_size, 7, 2, 24, width, height, 0, 0, 32, 0, 32, 32, bytes_per_line, 4,
0xFF0000, 0xFF00, 0xFF, 8, 256, ncolors, ncolors, width, height, 0, 0]
rows = b"".join(pixels[y * width * 4:(y + 1) * width * 4] + b"\0" * pad for y in range(height))
return struct.pack(">25I", *fields) + name + b"\0" * (12 * ncolors) + rows
class AssetChoice(unittest.TestCase):
RELEASES = [
{"tag_name": "preview", "prerelease": True, "assets": [
{"name": "DuckStation-x64-SSE2.AppImage", "browser_download_url": "u1"},
{"name": "DuckStation-x64.AppImage", "browser_download_url": "u2"}]},
{"tag_name": "latest", "prerelease": False, "assets": [
{"name": "DuckStation-x64.AppImage", "browser_download_url": "u3"}]},
]
def test_first_match_in_api_order_like_getreleaseurlgh(self) -> None:
asset = sp.select_asset(self.RELEASES, "AppImage", "x64.")
self.assertEqual((asset["tag"], asset["name"], asset["url"]),
("preview", "DuckStation-x64.AppImage", "u2"))
self.assertTrue(asset["prerelease"])
def test_no_match(self) -> None:
self.assertIsNone(sp.select_asset(self.RELEASES, "Qt.AppImage", ""))
def test_every_emulator_names_its_installer_and_profile(self) -> None:
profiles = {p.stem for p in (ROOT / "emulators").glob("*.yml")}
for name, emu in sp.EMULATORS.items():
self.assertEqual(name, emu.name)
self.assertIn(name, profiles)
self.assertIn("EmuDeck", emu.installer_ref)
self.assertTrue(emu.error_patterns)
class AppImage(unittest.TestCase):
def test_offset_is_the_end_of_the_section_headers(self) -> None:
head = bytearray(64)
head[:5] = b"\x7fELF\x02"
struct.pack_into("<Q", head, 0x28, 1000)
struct.pack_into("<HH", head, 0x3A, 64, 30)
self.assertEqual(sp.appimage_offset(bytes(head)), 1000 + 64 * 30)
def test_not_elf(self) -> None:
with self.assertRaises(cp.ProbeError):
sp.appimage_offset(b"hsqs" + bytes(60))
class Framebuffer(unittest.TestCase):
def test_reads_rgb_and_skips_colormap_and_padding(self) -> None:
# Two pixels, BGRX: red then blue.
pixels = bytes((0, 0, 255, 0, 255, 0, 0, 0))
width, height, rgb = sp.parse_xwd(xwd(2, 1, pixels, ncolors=3, pad=8))
self.assertEqual((width, height), (2, 1))
self.assertEqual(rgb, bytes((255, 0, 0, 0, 0, 255)))
def test_refuses_other_layouts(self) -> None:
data = bytearray(xwd(1, 1, bytes(4)))
struct.pack_into(">I", data, 44, 16)
with self.assertRaises(cp.ProbeError):
sp.parse_xwd(bytes(data))
def test_truncated(self) -> None:
with self.assertRaises(cp.ProbeError):
sp.parse_xwd(xwd(4, 4, bytes(64))[:-10])
class Settings(unittest.TestCase):
def test_bios_folder_and_overrides(self) -> None:
emu = sp.EMULATORS["duckstation"]
sections = sp.emulator_settings(emu, Path("/b"), [("GPU.Renderer", "Vulkan"), ("New.Key", "1")])
self.assertEqual(sections["BIOS"]["SearchDirectory"], "/b")
self.assertEqual(sections["GPU"]["Renderer"], "Vulkan")
self.assertEqual(sections["New"]["Key"], "1")
self.assertEqual(emu.settings["GPU"]["Renderer"], "Software")
text = sp.ini_text(sections)
self.assertIn("[BIOS]\nSearchDirectory = /b\n", text)
def test_override_needs_a_section(self) -> None:
with self.assertRaises(cp.ProbeError):
sp.emulator_settings(sp.EMULATORS["pcsx2"], Path("/b"), [("Renderer", "13")])
def test_display_wrapper_passes_the_command_through(self) -> None:
argv = sp.with_display(["/emu", "--", "game.cue"], Path("/fb"))
self.assertEqual(argv[:2], ["/bin/sh", "-c"])
self.assertEqual(argv[3:], ["xvfb", "/fb", "/emu", "--", "game.cue"])
self.assertIn(f'Xvfb ":{sp.DISPLAY_NUMBER}"', argv[2])
class Oracle(unittest.TestCase):
def test_emulator_patterns_join_the_oracle_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "oracle.yml"
path.write_text("error_patterns:\n - {pattern: 'shared', source_ref: x}\n"
"cores:\n duckstation:\n error_screens:\n"
" - {sha1: '" + "a" * 40 + "', shows: 'no BIOS'}\n"
"content:\n - sha1: '" + "b" * 40 + "'\n references:\n"
" duckstation:\n frames:\n"
" - {sha1: '" + "c" * 40 + "', stage: bios, shows: 'logo'}\n",
encoding="utf-8")
oracle = sp.emulator_oracle(sp.EMULATORS["duckstation"], path, "b" * 40)
self.assertEqual(oracle["references"][0]["sha1"], "c" * 40)
self.assertEqual(oracle["error_screens"][0]["shows"], "no BIOS")
self.assertEqual(oracle["error_patterns"][0], "shared")
self.assertIn(r"No\ BIOS\ image\ found\ for", oracle["error_patterns"])
self.assertEqual(len(oracle["sha256"]), 64)
def test_without_the_file_the_oracle_names_the_script(self) -> None:
oracle = sp.emulator_oracle(sp.EMULATORS["pcsx2"], Path("/nonexistent/oracle.yml"), "0" * 40)
self.assertEqual(oracle["file"], sp.SCRIPT)
self.assertEqual(oracle["references"], [])
def test_a_log_line_gives_bios_error_through_core_probe_judge(self) -> None:
oracle = sp.emulator_oracle(sp.EMULATORS["duckstation"], Path("/nonexistent"), "0" * 40)
rep = {"loaded": True, "observables": {"frame_hashes": [[0, "d" * 40]], "uniform_frames": []},
"logs": [{"frame": 3, "level": "info", "text": "E: No BIOS image found for NTSC-J region."}]}
verdict, evidence = cp.judge(rep, oracle)
self.assertEqual((verdict, evidence["rule"]), ("bios_error", "error_pattern"))
def test_an_early_exit_is_a_crash_and_a_black_tail_a_hang(self) -> None:
oracle = {"references": [], "error_patterns": [], "error_screens": []}
rep = {"loaded": False, "exit": {"status": 1, "signal": None}, "observables": {}, "logs": []}
self.assertEqual(cp.judge(rep, oracle)[0], "crash")
black = "e" * 40
rep = {"loaded": True, "logs": [], "observables": {
"frame_hashes": [[0, "f" * 40], [1, black], [2, black]], "uniform_frames": [black]}}
self.assertEqual(cp.judge(rep, oracle)[0], "hang")
def test_check_result_carries_the_verdicts(self) -> None:
def variant(label: str, verdict: str) -> dict:
return {"label": label, "flags": [], "verdict": verdict, "repetitions": [
{"loaded": True, "observables": {"verdict": verdict}}]}
result = sp.check_result([variant("absent", "bios_error"), variant("scph5501.bin", "boots")],
"verdict")
self.assertEqual(result["status"], "differs")
self.assertEqual(result["verdicts"], {"absent": "bios_error", "scph5501.bin": "boots"})
class HostLock(unittest.TestCase):
def test_a_run_holds_the_probe_lock_for_itself(self) -> None:
import contextlib
from unittest import mock
events = []
@contextlib.contextmanager
def lock():
events.append("take")
yield
events.append("free")
def body(_args, _emu):
events.append("run")
return 0
with tempfile.NamedTemporaryFile(suffix=".cue") as content, \
mock.patch.object(sp.cp, "host_probe_lock", lock), mock.patch.object(sp, "_probe", body):
args = sp.build_parser().parse_args(["run", "duckstation", "--content", content.name])
self.assertEqual(sp.probe(args), 0)
self.assertEqual(events, ["take", "run", "free"])
class CommandLine(unittest.TestCase):
def test_check_refuses_two_destinations(self) -> None:
with tempfile.NamedTemporaryFile(suffix=".cue") as content:
args = sp.build_parser().parse_args(
["check", "duckstation", "--content", content.name, "--vary", "a.bin=", "--vary", "b.bin=",
"--claim", "c", "--method", "m"])
self.assertIn("--vary must name the same DEST in every occurrence", sp.validate(args))
def test_unknown_observable(self) -> None:
with tempfile.NamedTemporaryFile(suffix=".cue") as content:
args = sp.build_parser().parse_args(
["check", "pcsx2", "--content", content.name, "--vary", "a.bin=", "--observable", "fps",
"--claim", "c", "--method", "m"])
self.assertTrue(any("--observable fps" in p for p in sp.validate(args)))
if __name__ == "__main__":
unittest.main()
tests/test_workflow_paths.py
+5 -1
View File
@@ -154,7 +154,11 @@ class ThePullRequestBuildsTheSite(unittest.TestCase):
if not path.startswith(("scripts/", ".github/"))
}
self.assertIn("wiki/**", inputs)
self.assertEqual(sorted(inputs - watched), [])
# A path the deploy names inside a folder validate.yml watches
# whole (tests/campaign/content.yml under tests/**) is watched.
unwatched = [path for path in inputs - watched
if not any(fnmatch.fnmatch(path, pattern) for pattern in watched)]
self.assertEqual(sorted(unwatched), [])
self.assertIn("scripts/**", watched)