kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 00:27:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge one-file measurement records

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 10:24:41 +01:00
2 parents a9301fbdc0 da68f19411
commit 0fbf920e45
23 files changed
+972 -1483
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitattributes
+4
View File
@@ -21,3 +21,7 @@ install.ps1 text eol=lf
# preserved, so they follow the LF rule above.
bios/** -text
data/** -text
# Measurement records are append-only: two branches that each add a line to
# the same file merge by keeping both.
measurements/*.jsonl text eol=lf merge=union
.github/workflows/deploy-site.yml
+2 -15
View File
@@ -57,7 +57,6 @@ on:
- "scripts/slots.py"
- "scripts/sourceref.py"
- "scripts/split_pack.py"
- "scripts/sync_measurements.py"
- "scripts/truth.py"
- "scripts/upstream.py"
- "scripts/validate_schemas.py"
@@ -72,7 +71,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
# The Starlight project: config, components, styles, lockfile. Its
# generated folders are gitignored and never trigger anything.
- "website/**"
@@ -118,18 +117,6 @@ jobs:
- run: npm ci --prefix website
# The core probe records live in Abdess/retrobios-measurements, read
# at the commit measurements-pin.json names and never at its head: a
# record reaches the site when the pin moves. One shallow fetch of that
# commit, with git alone.
- name: Clone the measurements at their pin
run: |
repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])")
commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])")
git init -q .cache/measurements
git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit"
git -C .cache/measurements checkout -q --detach "$commit"
- name: Validate data contracts
run: python scripts/validate_schemas.py
@@ -149,7 +136,7 @@ jobs:
- name: Generate site
run: |
python scripts/generate_site.py --require-measurements
python scripts/generate_site.py
python scripts/generate_readme.py --db database.json --platforms-dir platforms
# starlight-links-validator fails the build on a dead link or anchor.
npm run build --prefix website
.github/workflows/validate.yml
+3 -13
View File
@@ -29,7 +29,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
- "website/**"
# Spelled out twice because the workflow parser reads no YAML anchor. A test
# holds the two lists equal.
@@ -54,7 +54,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
- "website/**"
permissions:
@@ -261,21 +261,11 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# The measurement records at their pin, as deploy-site.yml reads them:
# the pages and the schema check read the same lines.
- name: Clone the measurements at their pin
run: |
repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])")
commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])")
git init -q .cache/measurements
git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit"
git -C .cache/measurements checkout -q --detach "$commit"
# The pages are gitignored, and the tests that read them fail
# in CI without it. The suite runs whatever this step returned, so a
# generator that stops still leaves every other test reporting.
- name: Generate the site
run: python scripts/generate_site.py --require-measurements
run: python scripts/generate_site.py
- name: Run test suite
if: ${{ !cancelled() }}
measurements-pin.json
-4
View File
@@ -1,4 +0,0 @@
{
"commit": "75e878143687201d1d7ca34f09cb43be3dc8d855",
"repo": "https://github.com/Abdess/retrobios-measurements"
}
measurements/beetle_saturn.jsonl
+6
View File
@@ -0,0 +1,6 @@
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads changes what the Saturn BIOS shows and does for a Video CD.","method":"Saturn BIOS v1.01 (Japan), region forced to Japan, Video CD Card option enabled, automatic RTC left on and the host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic, the authored Video CD inserted. 3000 frames; Saturn A (libretro B) held 15 frames from frame 1300, which is Play on the BIOS CD player, and Start 15 frames from frame 1900. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"8fe992a692603f9e0e4cefacc1a8348d17409463","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","mpeg_card.bin-absent","zeros_512k.bin","one_byte.bin"]}],"statement":"video_sha1 is identical across all 11 variants","status":"same"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"6e9a44698a2eee492e8d7eff67f2bd5634f52de8cc1af7885f334c3f29fcf2bd"},"emulator":"beetle_saturn","flags":["load average reached 7.02 (threshold 6.0)"],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[7.02,7.37,5.72],"load_before":[3.87,7.46,5.31],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-6afcf7b700ba","inputs":{"content":{"label":"authored Video CD: Big Buck Bunny 60 s, NTSC VCD 2.0 (identity.json beside the image)","sha1":"150cfb81d3c844f0721f2864073a33e326492d46","size":172,"tracks":[{"file":"bbb_vcd.bin","sha1":"e7952e60928dba7f3ceb6c3316894f96ad24e175","size":12169248}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":150,"frames":3000,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled","beetle_saturn_region":"Japan"},"presses":[{"button":0,"frame":1300,"length":15},{"button":3,"frame":1900,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T04:57:35+00:00","subject":"mpeg-card-vcd-bios-jp","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":3,"37":1,"40":2999,"45":4,"46":1,"47":3000,"51":1,"55":2,"57":1,"8":1,"87":1},"files_opened":[{"path":"sega_101.bin","sha1_after":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720","sha1_staged":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":330.3950935700897,"fps_nominal":59.82650314089141,"frames":3000,"input_pressed":60,"input_queries":156000,"last_frame_size":[330,224],"library_name":"Beetle Saturn","library_version":"v1.32.1","loaded":true,"logs":[{"frame":0,"level":"info","text":"Loading \"<content>/bbb_vcd.cue\""},{"frame":0,"level":"debug","text":"Disc 1"},{"frame":0,"level":"debug","text":"- Track 1, LBA: 0 DATA"},{"frame":0,"level":"debug","text":"- Track 2, LBA: 450 DATA"},{"frame":0,"level":"debug","text":"Leadout: 5174"},{"frame":0,"level":"info","text":"Calculating game ID (1 discs)"},{"frame":0,"level":"info","text":"Game ID is: "},{"frame":0,"level":"info","text":"CPU Cache Emulation Mode: Data only"},{"frame":0,"level":"info","text":"Region: 0x1."},{"frame":0,"level":"info","text":"Cart: Backup Memory."},{"frame":0,"level":"info","text":"Controller 1: Control Pad"},{"frame":0,"level":"info","text":"Controller 2: Control Pad"}],"ms_per_frame":3.0266793286621883,"observables":{"audio_sha1":"ebe47d681c9a51cfaa5c9228920e0b7b2c494139","frame_hashes":[[0,"4c96f9da044f34b3000d6392f31189293ec78ee9"],[150,"a189f70d4002cace490f589ea46402fa8444ea47"],[300,"5a4435c3aba5196bcb9ce7b3bcaaae4d49e8f97c"],[450,"5edec5180d1c04dc0ead1e5fbbc7d6d53cc23fdb"],[600,"4c96f9da044f34b3000d6392f31189293ec78ee9"],[750,"4c96f9da044f34b3000d6392f31189293ec78ee9"],[900,"4c96f9da044Line truncated
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads changes what the Saturn BIOS shows and does for a Video CD.","method":"Saturn BIOS v1.00a (mpr-17933), region forced to North America, Video CD Card option enabled, automatic RTC left on and the host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic, the authored Video CD inserted. 3000 frames; Saturn A (libretro B) held 15 frames from frame 1300, which is Play on the BIOS CD player, and Start 15 frames from frame 1900. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"6f95024fbc87ea46a22be37e04dabe558b900601","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","mpeg_card.bin-absent","zeros_512k.bin","one_byte.bin"]}],"statement":"video_sha1 is identical across all 11 variants","status":"same"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"6e9a44698a2eee492e8d7eff67f2bd5634f52de8cc1af7885f334c3f29fcf2bd"},"emulator":"beetle_saturn","flags":["load average reached 8.61 (threshold 6.0)"],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[8.61,7.5,6.41],"load_before":[3.72,6.03,5.7],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-ac66124f74ae","inputs":{"content":{"label":"authored Video CD: Big Buck Bunny 60 s, NTSC VCD 2.0 (identity.json beside the image)","sha1":"150cfb81d3c844f0721f2864073a33e326492d46","size":172,"tracks":[{"file":"bbb_vcd.bin","sha1":"e7952e60928dba7f3ceb6c3316894f96ad24e175","size":12169248}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":150,"frames":3000,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled","beetle_saturn_region":"North America"},"presses":[{"button":0,"frame":1300,"length":15},{"button":3,"frame":1900,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T05:07:13+00:00","subject":"mpeg-card-vcd-bios-us","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":3,"37":1,"40":2999,"45":4,"46":1,"47":3000,"51":1,"55":2,"57":1,"8":1,"87":1},"files_opened":[{"path":"mpr-17933.bin","sha1_after":"faa8ea183a6d7bbe5d4e03bb1332519800d3fbc3","sha1_staged":"faa8ea183a6d7bbe5d4e03bb1332519800d3fbc3"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":244.16338538684496,"fps_nominal":59.82650314089141,"frames":3000,"input_pressed":60,"input_queries":156000,"last_frame_size":[330,224],"library_name":"Beetle Saturn","library_version":"v1.32.1","loaded":true,"logs":[{"frame":0,"level":"info","text":"Loading \"<content>/bbb_vcd.cue\""},{"frame":0,"level":"debug","text":"Disc 1"},{"frame":0,"level":"debug","text":"- Track 1, LBA: 0 DATA"},{"frame":0,"level":"debug","text":"- Track 2, LBA: 450 DATA"},{"frame":0,"level":"debug","text":"Leadout: 5174"},{"frame":0,"level":"info","text":"Calculating game ID (1 discs)"},{"frame":0,"level":"info","text":"Game ID is: "},{"frame":0,"level":"info","text":"CPU Cache Emulation Mode: Data only"},{"frame":0,"level":"info","text":"Region: 0x4."},{"frame":0,"level":"info","text":"Cart: Backup Memory."},{"frame":0,"level":"info","text":"Controller 1: Control Pad"},{"frame":0,"level":"info","text":"Controller 2: Control Pad"}],"ms_per_frame":4.095618179669448,"observables":{"audio_sha1":"9c8887d4ee16309b6da657ede701a90a97f1e61a","frame_hashes":[[0,"4c96f9da044f34b3000d6392f31189293ec78ee9"],[150,"e9725385e2865e6448bbbc709fc72e54f5478569"],[300,"a87b28c1dca58cf41d1c5c892be3a14eb0c2f7ff"],[450,"f6f6ad7714da4b7c32a7736276c8cba7ef9d45d0"],[600,"4c96f9da044f34b3000d6392f31189293ec78ee9"],[750,"4c96f9da044f34b3000d6392f31189293ec78ee9Line truncated
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads changes what the Saturn BIOS shows and does for a Video CD.","method":"Experimental build, not what users run: mednafen/ss/cdb.c reads the AUTH_DEVICE and GET_AUTH device type from CR2 (CTR.CD[1]) instead of CR3 (CTR.CD[2]), plus a command trace to a file. Saturn BIOS v1.01 (Japan), region forced to Japan, Video CD Card option enabled, automatic RTC left on and the host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic, the authored Video CD inserted. 3000 frames; Saturn A (libretro B) held 15 frames from frame 1300, which is Play on the BIOS CD player, and Start 15 frames from frame 1900. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"88683d404f1266d0177c73f970cbd5996c11b7c1","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","zeros_512k.bin","one_byte.bin"]},{"value":"4283d78ac91cfef738daf1fe682dff4e531809e4","variants":["mpeg_card.bin-absent"]}],"statement":"video_sha1 takes 2 distinct values across 11 variants","status":"differs"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8 after editing mednafen/ss/cdb.c: AUTH_DEVICE and GET_AUTH read CTR.CD[1] instead of CTR.CD[2], plus a command trace","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"fc9c33c697ddcde7547f2315f6ff91231f2ecfa24280c884a0185b880ffc2dfb"},"emulator":"beetle_saturn","flags":[],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[2.41,4.58,5.69],"load_before":[3.24,7.0,6.58],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-43527b763ace","inputs":{"content":{"label":"authored Video CD: Big Buck Bunny 60 s, NTSC VCD 2.0 (identity.json beside the image)","sha1":"150cfb81d3c844f0721f2864073a33e326492d46","size":172,"tracks":[{"file":"bbb_vcd.bin","sha1":"e7952e60928dba7f3ceb6c3316894f96ad24e175","size":12169248}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":150,"frames":3000,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled","beetle_saturn_region":"Japan"},"presses":[{"button":0,"frame":1300,"length":15},{"button":3,"frame":1900,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T05:14:32+00:00","subject":"mpeg-card-vcd-bios-jp-cr2-build","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":3,"37":1,"40":2999,"45":4,"46":1,"47":3000,"51":1,"55":2,"57":1,"8":1,"87":1},"files_opened":[{"path":"sega_101.bin","sha1_after":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720","sha1_staged":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":337.8442323774398,"fps_nominal":59.82650314089141,"frames":3000,"input_pressed":60,"input_queries":156000,"last_frame_size":[330,224],"library_name":"Beetle Saturn","library_version":"v1.32.1","loaded":true,"logs":[{"frame":0,"level":"info","text":"Loading \"<content>/bbb_vcd.cue\""},{"frame":0,"level":"debug","text":"Disc 1"},{"frame":0,"level":"debug","text":"- Track 1, LBA: 0 DATA"},{"frame":0,"level":"debug","text":"- Track 2, LBA: 450 DATA"},{"frame":0,"level":"debug","text":"Leadout: 5174"},{"frame":0,"level":"info","text":"Calculating game ID (1 discs)"},{"frame":0,"level":"info","text":"Game ID is: "},{"frame":0,"level":"info","text":"CPU Cache Emulation Mode: Data only"},{"frame":0,"level":"info","text":"Region: 0x1."},{"frame":0,"level":"info","text":"Cart: Backup Memory."},{"frame":0,"level":"info","text":"Controller 1: Control Pad"},{"frame":0,"level":"info","text":"Controller 2: Control Pad"}],"ms_per_frame":2.9599439746622616,"observables":{"audio_sha1":"85f24b268528b103056d511Line truncated
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads changes what the Saturn BIOS shows and does for a Video CD.","method":"Saturn BIOS v1.01 (Japan), region forced to Japan, Video CD Card option enabled, automatic RTC left on and the host clock running (no libfaketime), to test whether runs repeat, the authored Video CD inserted. 3000 frames; Saturn A (libretro B) held 15 frames from frame 1300, which is Play on the BIOS CD player, and Start 15 frames from frame 1900. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"bde95c486614dc797ba5a939fb738d947b479fa8","variants":["mpeg_card.bin"]},{"value":"5a14993408a486a4c55341ca7284f89e148dfc18","variants":["mpeg_card.bin.3fee63d4"]},{"value":"7e3d403552c31c494beb9847262bf86cf266c3fa","variants":["mpeg_card.bin.80179c49"]},{"value":"b639d8c3130638876b398c66f463b98f535a2497","variants":["mpeg_card.bin.821af2bd"]},{"value":"5dd8db90027d8ef7670c0805751b8973920195a9","variants":["mpeg_card.bin.b60e8866"]},{"value":"fabe749bf34c09de4c26d4328e2744876488a18e","variants":["mpeg_card.bin.b7450812"]},{"value":"ef9d0fab7815222344d7d669bd0dbc6e0d811174","variants":["mpeg_card.bin.ee821d3f"]},{"value":"164f2e824cc15672226bebc07998f40d5e3422ce","variants":["mpeg_card.bin.ef9de845"]},{"value":"1c920f21c6fc3d402fa45a7a5fb4d302a82d4898","variants":["mpeg_card.bin-absent"]},{"value":"2a6d6e171b0bafe5b60ec41a37db4322ff2b12e8","variants":["zeros_512k.bin"]},{"value":"2d5be7af25846153e20081a8b12b614ff348c4b9","variants":["one_byte.bin"]}],"statement":"video_sha1 is not settled in 11 of 11 variants (mpeg_card.bin, mpeg_card.bin.3fee63d4, mpeg_card.bin.80179c49, mpeg_card.bin.821af2bd, mpeg_card.bin.b60e8866, mpeg_card.bin.b7450812, mpeg_card.bin.ee821d3f, mpeg_card.bin.ef9de845, mpeg_card.bin-absent, zeros_512k.bin, one_byte.bin): no comparison is drawn","status":"inconclusive"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"6e9a44698a2eee492e8d7eff67f2bd5634f52de8cc1af7885f334c3f29fcf2bd"},"emulator":"beetle_saturn","flags":["mpeg_card.bin: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.3fee63d4: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.80179c49: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.821af2bd: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.b60e8866: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.b7450812: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.ee821d3f: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin.ef9de845: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","mpeg_card.bin-absent: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","zeros_512k.bin: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1","one_byte.bin: repetition 1 disagrees with repetition 0 on frame_hashes, frame_sha1, state_sha1, video_sha1"],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[1.77,2.58,4.32],"load_before":[1.81,3.2,4.89],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-bf3d8283b265","inputs":{"content":{"label":"authored Video CD: Big Buck Bunny 60 s, NTSC VCD 2.0 (identity.json beside the image)","sha1":"150cfb81d3c844f0721f2864073a33e326492d46","size":172,"tracks":[{"file":"bbb_vcd.bin","sha1":"e7952e60928dba7f3ceb6c3316894f96ad24e175","size":12169248}]},"env":{},"frame_hash_every":150,"frames":3000,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled","beetle_saturn_region":"Japan"},"presses":[{"button":0,"frame":1300,"length":15},{"button":3,"frame":1900,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"recorLine truncated
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads decides whether Lunar - Silver Star Story MPEG (T-27904G) detects the Video CD Card and plays its movies.","method":"Saturn BIOS v1.01 (Japan), Video CD Card option enabled, region auto-detected from the disc (Japan), host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic. 4800 frames; Start held 15 frames at frames 2400, 3000, 3600 and 4200. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"e248465e55f8edbdfb16a8ca6588817ffa9d8064","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","mpeg_card.bin-absent","zeros_512k.bin","one_byte.bin"]}],"statement":"video_sha1 is identical across all 11 variants","status":"same"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"6e9a44698a2eee492e8d7eff67f2bd5634f52de8cc1af7885f334c3f29fcf2bd"},"emulator":"beetle_saturn","flags":[],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[3.16,2.62,3.83],"load_before":[1.77,2.58,4.32],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-5d8f0b375830","inputs":{"content":{"label":"Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00 (cue sha1 7a609a9b579de648197ccfb5ab7bf4b3beb07d95)","sha1":"7a609a9b579de648197ccfb5ab7bf4b3beb07d95","size":554,"tracks":[{"file":"Lunar - Silver Star Story Complete (Japan) (Track 1).bin","sha1":"b6d71e1c96211af46d83278eccf7d6827a57d004","size":83710032},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 2).bin","sha1":"c2406d2953cec7f9fe5826aa71c7359c2baeb3a7","size":491934912},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 3).bin","sha1":"71c3bcfc22038d0f25126b34f703aa464259cfb5","size":25867296},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 4).bin","sha1":"b9c73d3890cd699cb1718131c2f7f04897a14d65","size":1872192}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":300,"frames":4800,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled"},"presses":[{"button":3,"frame":2400,"length":15},{"button":3,"frame":3000,"length":15},{"button":3,"frame":3600,"length":15},{"button":3,"frame":4200,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T05:26:27+00:00","subject":"mpeg-card-lunar","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":3,"37":1,"40":4799,"45":4,"46":1,"47":4800,"51":1,"55":2,"57":1,"8":1,"87":1},"files_opened":[{"path":"sega_101.bin","sha1_after":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720","sha1_staged":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":366.16169342725783,"fps_nominal":59.82650314089141,"frames":4800,"input_pressed":120,"input_queries":249600,"last_frame_size":[330,224],"library_name":"Beetle Saturn","library_version":"v1.32.1","loaded":true,"logs":[{"frame":0,"level":"info","text":"Loading \"<content>/Lunar - Silver Star Story Complete (Japan).cue\""},{"frame":0,"level":"debug","text":"Disc 1"},{"frame":0,"level":"debug","text":"- Track 1, LBA: 0 DATA"},{"frame":0,"level":"debug","text":"- Track 2, LBA: 35891 DATA"},{"frame":0,"level":"debug","text":"- Track 3, LBA: 244897 AUDIO"},{"frame":0,"level":"debug","text":"- Track 4, LBA: 255894 AUDIO"},{"frame":0,"level":"debug","text":"Leadout: 256541"},{"frame":0,"level":"info","text":"Calculating game ID (1 discs)"},{"frame":0,"level":"info","text":"Game ID is: T-27904G"},{"frame":0,"level":"info","text":"This is a Saturn disc."},{"frame":0,"level":"info","text":"Disc Region: \"Japan\""},{"frame":0,"level":"inLine truncated
{"check":{"claim":"The mpeg_card.bin image Beetle Saturn loads decides whether Lunar - Silver Star Story MPEG (T-27904G) detects the Video CD Card and plays its movies.","method":"Experimental build, not what users run: AUTH_DEVICE and GET_AUTH read the device type from CR2. Saturn BIOS v1.01 (Japan), Video CD Card option enabled, region auto-detected from the disc (Japan), host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic. 4800 frames; Start held 15 frames at frames 2400, 3000, 3600 and 4200. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"f7db10347ce2081c188f83fbe5640131b4f2ceb5","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","zeros_512k.bin","one_byte.bin"]},{"value":"d60fcae489df0c317cca4e95f6f57c323be20ee4","variants":["mpeg_card.bin-absent"]}],"statement":"video_sha1 takes 2 distinct values across 11 variants","status":"differs"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8 after editing mednafen/ss/cdb.c: AUTH_DEVICE and GET_AUTH read CTR.CD[1] instead of CTR.CD[2], plus a command trace","commit":"2ce77e20a956dc9a67b8d3d527f477b984310feb","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"mednafen_saturn_libretro.so","library_name":"Beetle Saturn","library_version":"v1.32.1","repo":"https://github.com/libretro/beetle-saturn-libretro","sha256":"fc9c33c697ddcde7547f2315f6ff91231f2ecfa24280c884a0185b880ffc2dfb"},"emulator":"beetle_saturn","flags":[],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[3.24,2.85,3.56],"load_before":[3.16,2.62,3.83],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-f532e5006714","inputs":{"content":{"label":"Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00 (cue sha1 7a609a9b579de648197ccfb5ab7bf4b3beb07d95)","sha1":"7a609a9b579de648197ccfb5ab7bf4b3beb07d95","size":554,"tracks":[{"file":"Lunar - Silver Star Story Complete (Japan) (Track 1).bin","sha1":"b6d71e1c96211af46d83278eccf7d6827a57d004","size":83710032},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 2).bin","sha1":"c2406d2953cec7f9fe5826aa71c7359c2baeb3a7","size":491934912},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 3).bin","sha1":"71c3bcfc22038d0f25126b34f703aa464259cfb5","size":25867296},{"file":"Lunar - Silver Star Story Complete (Japan) (Track 4).bin","sha1":"b9c73d3890cd699cb1718131c2f7f04897a14d65","size":1872192}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":300,"frames":4800,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","state:MPEG.AuthState","state:MPEG.Present","frame_hashes"],"options":{"beetle_saturn_mpeg_card":"enabled"},"presses":[{"button":3,"frame":2400,"length":15},{"button":3,"frame":3000,"length":15},{"button":3,"frame":3600,"length":15},{"button":3,"frame":4200,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T05:31:33+00:00","subject":"mpeg-card-lunar-cr2-build","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":3,"37":1,"40":4799,"45":4,"46":1,"47":4800,"51":1,"55":2,"57":1,"8":1,"87":1},"files_opened":[{"path":"sega_101.bin","sha1_after":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720","sha1_staged":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":363.57766936404164,"fps_nominal":59.82650314089141,"frames":4800,"input_pressed":120,"input_queries":249600,"last_frame_size":[330,224],"library_name":"Beetle Saturn","library_version":"v1.32.1","loaded":true,"logs":[{"frame":0,"level":"info","text":"Loading \"<content>/Lunar - Silver Star Story Complete (Japan).cue\""},{"frame":0,"level":"debug","text":"Disc 1"},{"frame":0,"level":"debug","text":"- Track 1, LBA: 0 DATA"},{"frame":0,"level":"debug","text":"- Track 2, LBA: 35891 DATA"},{"frame":0,"level":"debug","text":"- Track 3, LBA: 244897 AUDIO"},{"frame":0,"level":"debug","text":"- Track 4, LBA: 255894 AUDIO"},{"frame":0,"level"Line truncated
measurements/yabause.jsonl
+1
View File
@@ -0,0 +1 @@
{"check":{"claim":"The mpeg_card.bin image Yabause hands to the Saturn BIOS through the Get MPEG ROM command (0xE2) changes what the BIOS shows and does for a Video CD.","method":"Experimental build, not what RetroArch users run: libretro Yabause with yinit.mpegpath pointed at the system directory, which reaches the same cs2.c Cs2GetMPEGRom path that standalone Yabause uses when its MpegROM/Path setting is filled. Saturn BIOS v1.01 (Japan), one emulation thread, host clock frozen at 2000-01-01 00:00:00 by libfaketime 0.9.10 so runs are deterministic, the authored Video CD inserted. 3000 frames; the button the core maps to Saturn A held 15 frames from frame 1300 (Play on the BIOS CD player) and Start 15 frames from frame 1900. Repeated with each mpeg_card.bin image, with no file, with 512 KiB of zeros and with a 1-byte file; every other input is identical. Each variant runs twice in fresh processes.","observable":"video_sha1","result":{"groups":[{"value":"c89177305da310bd329a17d0f4a5e60ffe5cdece","variants":["mpeg_card.bin","mpeg_card.bin.3fee63d4","mpeg_card.bin.80179c49","mpeg_card.bin.821af2bd","mpeg_card.bin.b60e8866","mpeg_card.bin.b7450812","mpeg_card.bin.ee821d3f","mpeg_card.bin.ef9de845","mpeg_card.bin-absent","zeros_512k.bin","one_byte.bin"]}],"statement":"video_sha1 is identical across all 11 variants","status":"same"},"variable":"mpeg_card.bin"},"core":{"build_command":"make -j8 in yabause/src/libretro, after setting yinit.mpegpath to <system>/mpeg_card.bin in libretro.c (the port sets it to NULL)","commit":"8926b0c6c347f8c5c755911ddb0ac695420ffbf8","compiler":["GCC: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0"],"file":"yabause_libretro.so","library_name":"Yabause","library_version":"v0.9.15 625bee35c","repo":"https://github.com/libretro/yabause","sha256":"1530b0a3c7f2ff762e4b02e22f016be6753a7f3864306cf17ca810bccee0e5c4"},"emulator":"yabause","flags":[],"frontend":{"commit":"8b12685c1652732828c146336a14a5fc265fffd3","script":"scripts/core_probe.py","script_modified":false,"script_sha256":"a95ab54a8d8f9ecc5dc2a55fd0ab93ffc66fe6466e2d89eea56a1a95c17d858b"},"host":{"cpu_count":12,"cpu_model":"13th Gen Intel(R) Core(TM) i9-13900K","kernel":"6.18.54.1-microsoft-standard-WSL2","load_after":[1.81,3.2,4.89],"load_before":[2.41,4.58,5.69],"machine":"x86_64","max_load":6.0,"os":"Linux-6.18.54.1-microsoft-standard-WSL2-x86_64-with-glibc2.39","python":"3.12.3"},"id":"2026-10-27-92c9da074bcb","inputs":{"content":{"label":"authored Video CD: Big Buck Bunny 60 s, NTSC VCD 2.0 (identity.json beside the image)","sha1":"150cfb81d3c844f0721f2864073a33e326492d46","size":172,"tracks":[{"file":"bbb_vcd.bin","sha1":"e7952e60928dba7f3ceb6c3316894f96ad24e175","size":12169248}]},"env":{"FAKETIME":"2000-01-01 00:00:00","FAKETIME_DONT_FAKE_MONOTONIC":"1","LD_PRELOAD":"libfaketime.so.1 sha256:7ac8a1cdff1b627548de11513fddedd75014f5b041e9ae9054a634bb04aa7982"},"frame_hash_every":150,"frames":3000,"joypad_ports":2,"observables":["frame_sha1","video_sha1","audio_sha1","state_sha1","frame_hashes"],"options":{"yabause_numthreads":"1"},"presses":[{"button":0,"frame":1300,"length":15},{"button":3,"frame":1900,"length":15}],"repetitions":2,"warmup":0},"kind":"check","migrated_from_version":1,"record_version":3,"recorded_at":"2026-10-27T05:18:19+00:00","subject":"mpeg-card-vcd-bios-jp-mpegpath-build","supersedes":null,"variants":[{"flags":[],"label":"mpeg_card.bin","repetitions":[{"env_unanswered":{"11":1,"13":1,"28":1,"35":1,"36":1,"42":1,"44":1,"49":3000,"51":1,"57":1,"8":1},"files_opened":[{"path":"sega_101.bin","sha1_after":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720","sha1_staged":"df94c5b4d47eb3cc404d88b33a8fda237eaf4720"},{"path":"mpeg_card.bin","sha1_after":"2f5ebd434f6d5b82e93d03e5626007bd5380f652","sha1_staged":"2f5ebd434f6d5b82e93d03e5626007bd5380f652"}],"files_probed_absent":[],"fps":298.7348625172993,"fps_nominal":60.0,"frames":3000,"input_pressed":30,"input_queries":78000,"last_frame_size":[320,224],"library_name":"Yabause","library_version":"v0.9.15 625bee35c","loaded":true,"logs":[{"frame":0,"level":"warn","text":"<system>/saturn_bios.bin NOT FOUND"},{"frame":0,"level":"debug","text":"Starting emulation"},{"frame":0,"level":"debug","text":"Enabling cheevos"}],"ms_per_frame":3.3474499480022737,"observables":{"audio_sha1":"8eefdabcbbb9f662de4b0d04087df5bd18ba6e45","frame_hashes":[[0,"d87e1556593b17c10facada78fe3b3a35c0f7552"],[150,"b2f9f19e66b8981bf1547e970a8a3f6637b6934e"],[300,"06d859bca35a5194fbb8b4300dab66643988629f"],[450,"b6064ea4357cf541eac46afc7f167ed7878aab1c"],[600,"d87e1556593b17c10facada78fe3b3a35c0f7552"],[750,"cbe0871893b854ec3a8ae72b5f1be8bed18a5308"],[900,"6d9c3c739167faef80b27a5e88eeefe46a395264"],[1050,"6c20cfa2ca401bbbab7a92aa4a37910b54baa70c"],[1200,"7471401fb6d263292f57f8164a5c34994485c205"],[1350,"d939ec9ce85638cb231ac6d9703b0519e484c483"],[1500,"7915940919a745bede34db6d4fc34d975894a6eb"],[1650,"8859e95bce7870e8417ff0e430a38b96d338b115"],[1800,"50283a9425d3ff49a41cfbff6c9cce958ad93a54"],[1950,"42c55858c8eLine truncated
schemas/core-probe-data.schema.json
-204
View File
@@ -1,204 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-data.schema.json",
"title": "RetroBIOS core probe record data",
"description": "The sidecar of a core probe record, data/<emulator>/<id>.json.gz: every repetition in full, the raw timing values and compared values too large for the record line. With the record it rebuilds the full result.",
"type": "object",
"additionalProperties": false,
"required": [
"variants",
"timing_raw",
"group_values"
],
"properties": {
"variants": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"label",
"repetitions"
],
"properties": {
"label": {
"type": "string",
"minLength": 1
},
"repetitions": {
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/$defs/repetition"
}
}
}
}
},
"timing_raw": {
"type": "object",
"additionalProperties": {
"type": "array",
"minItems": 1,
"items": {
"type": "number"
}
}
},
"group_values": {
"type": "object",
"propertyNames": {
"pattern": "^[0-9a-f]{40}$"
}
}
},
"$defs": {
"sha1": {
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
"repetition": {
"type": "object",
"required": [
"loaded",
"library_name",
"logs"
],
"properties": {
"loaded": {
"type": "boolean"
},
"library_name": {
"type": "string"
},
"library_version": {
"type": "string"
},
"fps_nominal": {
"type": "number"
},
"frames": {
"type": "integer",
"minimum": 1
},
"warmup": {
"type": "integer",
"minimum": 0
},
"wall_s": {
"type": "number",
"minimum": 0
},
"ms_per_frame": {
"type": "number",
"minimum": 0
},
"fps": {
"type": "number",
"minimum": 0
},
"observables": {
"type": "object"
},
"logs": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"frame",
"level",
"text"
],
"properties": {
"frame": {
"type": "integer",
"minimum": 0
},
"level": {
"type": "string"
},
"text": {
"type": "string"
}
}
}
},
"options_declared": {
"type": "object"
},
"env_unanswered": {
"type": "object"
},
"last_frame_size": {
"type": "array",
"items": {
"type": "integer"
}
},
"tracing": {
"type": "string"
},
"input_queries": {
"type": "integer",
"minimum": 0
},
"input_pressed": {
"type": "integer",
"minimum": 0
},
"files_opened": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"path",
"sha1_after",
"sha1_staged"
],
"properties": {
"path": {
"type": "string"
},
"sha1_after": {
"$ref": "#/$defs/sha1"
},
"sha1_staged": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/$defs/sha1"
}
]
}
}
}
}
]
},
"files_probed_absent": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
}
}
}
}
}
schemas/core-probe-record.schema.json
+166 -117
View File
@@ -2,7 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json",
"title": "RetroBIOS core probe record",
"description": "One line of records/<emulator>.jsonl in the measurements repository: one run of scripts/core_probe.py, a libretro core executed headless, with what is needed to judge and reproduce it. The bulky rest (every repetition, core logs, raw timings) is the gzip sidecar data/<emulator>/<id>.json.gz, valid against core-probe-data.schema.json, whose sha256 is in `data`.",
"description": "One line of measurements/<emulator>.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, in full, with what is needed to judge and reproduce it. Every repetition, its core logs and per-frame hashes, and the raw timing values are in the line.",
"type": "object",
"additionalProperties": false,
"required": [
@@ -18,12 +18,11 @@
"host",
"inputs",
"variants",
"flags",
"data"
"flags"
],
"properties": {
"record_version": {
"const": 2
"const": 3
},
"id": {
"type": "string",
@@ -32,7 +31,7 @@
},
"emulator": {
"$ref": "#/$defs/slug",
"description": "The emulator profile the record is filed under: records/<emulator>.jsonl."
"description": "The emulator profile the record is filed under: measurements/<emulator>.jsonl."
},
"subject": {
"$ref": "#/$defs/slug",
@@ -361,13 +360,8 @@
"required": [
"label",
"system_files",
"flags",
"repetitions",
"loaded",
"observables",
"files_opened",
"files_probed_absent",
"tracing"
"flags"
],
"properties": {
"label": {
@@ -387,80 +381,18 @@
]
}
},
"repetitions": {
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/$defs/repetition"
}
},
"flags": {
"type": "array",
"items": {
"type": "string"
}
},
"repetitions": {
"type": "integer",
"minimum": 1,
"description": "Repetitions run. Each is in the sidecar."
},
"loaded": {
"type": "boolean",
"description": "Every repetition loaded the content."
},
"observables": {
"type": "object",
"description": "The first repetition's observables, frame_hashes left to the sidecar."
},
"files_opened": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"path",
"sha1_after",
"sha1_staged"
],
"properties": {
"path": {
"type": "string"
},
"sha1_after": {
"$ref": "#/$defs/sha1"
},
"sha1_staged": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/$defs/sha1"
}
]
}
}
}
}
]
},
"files_probed_absent": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
},
"tracing": {
"type": [
"string",
"null"
]
}
}
}
@@ -541,6 +473,7 @@
"type": "object",
"additionalProperties": false,
"required": [
"value",
"variants"
],
"properties": {
@@ -551,47 +484,13 @@
"items": {
"type": "string"
}
},
"value_sha1": {
"$ref": "#/$defs/sha1",
"description": "Stands for a list or object value kept in the sidecar's group_values."
}
},
"oneOf": [
{
"required": [
"value"
]
},
{
"required": [
"value_sha1"
]
}
]
}
}
}
}
}
}
},
"data": {
"type": "object",
"additionalProperties": false,
"required": [
"sha256",
"bytes"
],
"description": "The sidecar data/<emulator>/<id>.json.gz: gzip (no name, mtime 0) of canonical JSON.",
"properties": {
"sha256": {
"$ref": "#/$defs/sha256"
},
"bytes": {
"type": "integer",
"minimum": 1
}
}
}
},
"allOf": [
@@ -666,7 +565,8 @@
"mean",
"stdev",
"min",
"max"
"max",
"raw"
],
"properties": {
"median": {
@@ -684,13 +584,162 @@
},
"max": {
"type": "number"
},
"raw": {
"type": "array",
"minItems": 1,
"items": {
"type": "number"
}
}
},
"description": "Statistics over the repetitions. The raw values are in the sidecar."
"description": "Statistics over the repetitions, with the values they were computed from."
},
"slug": {
"type": "string",
"pattern": "^[a-z0-9]+(?:[-_][a-z0-9]+)*$"
},
"repetition": {
"type": "object",
"required": [
"loaded",
"library_name",
"logs"
],
"properties": {
"loaded": {
"type": "boolean"
},
"library_name": {
"type": "string"
},
"library_version": {
"type": "string"
},
"fps_nominal": {
"type": "number"
},
"frames": {
"type": "integer",
"minimum": 1
},
"warmup": {
"type": "integer",
"minimum": 0
},
"wall_s": {
"type": "number",
"minimum": 0
},
"ms_per_frame": {
"type": "number",
"minimum": 0
},
"fps": {
"type": "number",
"minimum": 0
},
"observables": {
"type": "object"
},
"logs": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"frame",
"level",
"text"
],
"properties": {
"frame": {
"type": "integer",
"minimum": 0
},
"level": {
"type": "string"
},
"text": {
"type": "string"
}
}
}
},
"options_declared": {
"type": "object"
},
"env_unanswered": {
"type": "object"
},
"last_frame_size": {
"type": "array",
"items": {
"type": "integer"
}
},
"tracing": {
"type": "string"
},
"input_queries": {
"type": "integer",
"minimum": 0
},
"input_pressed": {
"type": "integer",
"minimum": 0
},
"files_opened": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"path",
"sha1_after",
"sha1_staged"
],
"properties": {
"path": {
"type": "string"
},
"sha1_after": {
"$ref": "#/$defs/sha1"
},
"sha1_staged": {
"oneOf": [
{
"type": "null"
},
{
"$ref": "#/$defs/sha1"
}
]
}
}
}
}
]
},
"files_probed_absent": {
"oneOf": [
{
"type": "null"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
}
}
}
}
}
scripts/check_freshness.py
+3 -31
View File
@@ -3,7 +3,7 @@
Each layer the repository transcribes moves on its own schedule: platform
lists, hardware targets, core-info, buildbot system assets, dump catalogs,
romset DATs, the measurements pin, the CI toolchain. This script asks each of them the same
romset DATs, the CI toolchain. This script asks each of them the same
question, is the local copy the one upstream serves, and answers with one
row per subject.
@@ -40,7 +40,7 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterator
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
@@ -80,8 +80,7 @@ INSTALLERS = ("install.sh", "install.ps1")
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "measurements", "ci",
"profiles")
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED"
# Every status a finding may carry, in the order the summary counts them.
STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK)
@@ -915,31 +914,6 @@ def _check_buildbot_system() -> Finding:
detail=f"{names}{more} (_data_dirs.yml)", items=every)
# --- measurements -----------------------------------------------------------
def check_measurements(offline: bool, head: Callable[[str], str] | None = None,
pin_path: Path | None = None) -> Finding:
"""The measurements pin against the head of the measurements repository.
A record added there reaches the site only once the pin moves to it.
"""
import sync_measurements # noqa: PLC0415
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
local = pin["commit"][:12]
if offline:
return Finding("measurements", "records pin", SKIPPED, local=local, detail="offline")
try:
remote = (head or sync_measurements.remote_head)(pin["repo"])
except (sync_measurements.MeasurementsError, OSError, subprocess.TimeoutExpired) as exc:
return Finding("measurements", "records pin", UNKNOWN, local=local, detail=str(exc))
if remote == pin["commit"]:
return Finding("measurements", "records pin", OK, local=local, remote=remote[:12])
return Finding("measurements", "records pin", STALE, local=local, remote=remote[:12],
detail="newer records: python scripts/sync_measurements.py --fetch --bump")
# --- catalogs and recipes ---------------------------------------------------
@@ -1532,8 +1506,6 @@ def main() -> int:
findings.extend(check_data(args.offline))
if "catalogs" in areas:
findings.extend(check_catalogs(args.cache_dir, args.offline))
if "measurements" in areas:
findings.append(check_measurements(args.offline))
if "ci" in areas:
findings.extend(check_ci(args.cache_dir, args.offline))
if "profiles" in areas:
scripts/core_probe.py
+13 -48
View File
@@ -10,10 +10,9 @@ Three modes share one runner:
Each repetition runs in its own process, under strace when it is installed,
so the files a core opened are read from the kernel rather than guessed.
`--record DIR` appends one record per invocation to a clone of the
measurements repository, carrying what is needed to reproduce it (core build,
frontend commit, host, inputs by hash). measurement_records.py splits it into
a slim line and a gzip sidecar.
`--record` appends one record per invocation, as one line of
measurements/<emulator>.jsonl in this repository, carrying what is needed to
reproduce it (core build, frontend commit, host, inputs by hash).
"""
from __future__ import annotations
@@ -39,7 +38,6 @@ from dataclasses import dataclass, field
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
MEASUREMENTS_CLONE = ROOT / ".cache" / "measurements"
MIN_BENCH_REPETITIONS = 5
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1")
@@ -771,10 +769,9 @@ def build_parser() -> argparse.ArgumentParser:
common.add_argument("--core-repo", help="source repository URL of the core build")
common.add_argument("--core-commit", help="commit the core was built from")
common.add_argument("--build-command", help="command and flags that built the core")
common.add_argument("--record", metavar="DIR",
help="append the record to the measurements clone DIR "
f"(records/<emulator>.jsonl and data/<emulator>/), "
f"for example {MEASUREMENTS_CLONE.relative_to(ROOT)}")
common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
help="append the record as one line of measurements/<emulator>.jsonl "
"in the checkout DIR (default: this one)")
common.add_argument("--emulator", help="profile the record is filed under (default: the "
"profile whose cores: names the .so, else the "
"core's library name)")
@@ -823,7 +820,6 @@ def validate_args(args: argparse.Namespace) -> list[str]:
if args.png_dir and not args.frame_hash_every:
problems.append("--png-dir needs --frame-hash-every")
if args.record:
problems.extend(record_target_problems(Path(args.record)))
if not args.emulator and Path(args.core).is_file():
try:
default_emulator(Path(args.core), None)
@@ -907,32 +903,6 @@ def check_result(variants: list[dict], observable: str) -> dict:
}
def record_target_problems(directory: Path) -> list[str]:
"""Why records may not be written under ``directory``.
Records live in their own repository. A directory inside this one's
tracked tree would put them in its history, megabytes per campaign;
an ignored directory (tmp/, the clone under .cache/) is fine.
"""
target = directory.resolve()
if target != ROOT and ROOT not in target.parents:
return []
try:
ignored = subprocess.run(["git", "check-ignore", "-q", str(target)], cwd=ROOT,
capture_output=True, check=False).returncode
except OSError as exc:
return [f"--record {directory}: cannot ask git whether it is tracked ({exc})"]
if ignored == 0:
return []
if ignored != 1:
# 128: not a git checkout, nothing to keep records out of.
return []
return [f"--record {directory}: inside the tracked tree of this repository. Records go "
f"to the measurements repository: clone it with "
f"`python scripts/sync_measurements.py --fetch` and pass "
f"--record {MEASUREMENTS_CLONE.relative_to(ROOT)}"]
def default_emulator(core_file: Path, library_name: str | None) -> str:
"""The profile whose cores: names the .so, else the library name as a slug."""
stem = core_file.name.split(".")[0].removesuffix("_libretro")
@@ -966,24 +936,19 @@ def default_subject(record: dict) -> str:
return f"{record['kind']}-{slugify(content, 30)}-{digest}"
def write_record(directory: Path, full: dict) -> tuple[Path, Path]:
"""Append the record to its emulator's .jsonl and write its sidecar."""
from measurement_records import ( # noqa: PLC0415
line_text, parse_lines, records_path, seal, sidecar_path, split)
def write_record(directory: Path, full: dict) -> Path:
"""Append the record, with its id, to its emulator's .jsonl."""
from measurement_records import line_text, parse_lines, records_path, seal # noqa: PLC0415
slim, sidecar = split(full)
sealed, blob = seal(slim, sidecar)
sealed = seal(full)
lines = directory / records_path(sealed["emulator"])
existing = parse_lines(lines.read_text(encoding="utf-8"), str(lines)) if lines.is_file() else []
if sealed["supersedes"] and sealed["supersedes"] not in {r["id"] for r in existing}:
raise ProbeError(f"--supersedes {sealed['supersedes']}: no such record in {lines}")
data = directory / sidecar_path(sealed["emulator"], sealed["id"])
data.parent.mkdir(parents=True, exist_ok=True)
data.write_bytes(blob)
lines.parent.mkdir(parents=True, exist_ok=True)
with lines.open("a", encoding="utf-8") as handle:
handle.write(line_text(sealed) + "\n")
return lines, data
return lines
def probe(args: argparse.Namespace) -> int:
@@ -1060,11 +1025,11 @@ def probe(args: argparse.Namespace) -> int:
record["subject"] = args.subject or default_subject(record)
if args.record:
try:
lines, data = write_record(Path(args.record), record)
lines = write_record(Path(args.record), record)
except ProbeError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))}) and {data}",
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))})",
file=sys.stderr)
print(json.dumps(record, indent=2, sort_keys=False))
return 0
scripts/generate_site.py
+4 -10
View File
@@ -110,7 +110,6 @@ from siteguides import (
)
import release_record
import sitemeasurements
import sync_measurements
# The Starlight project, and where its pages go inside it.
STARLIGHT_DIR = "website"
@@ -11007,11 +11006,6 @@ def main() -> None:
help="directory the pages are written to (default: the project's "
"content collection, src/content/docs)",
)
parser.add_argument(
"--require-measurements", action="store_true",
help="fail when the measurements clone does not hold the pinned commit "
"(the deploy build passes it; without it the site lists no measurements)",
)
args = parser.parse_args()
if args.docs_dir is None:
args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
@@ -11168,9 +11162,9 @@ def _generate(args: argparse.Namespace) -> None:
if clash:
raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}")
print("Compiling the measurements at their pin...")
measured = sync_measurements.build(
public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements,
print("Compiling the measurements...")
measured = sitemeasurements.build(
public, emulators_dir=Path(args.emulators_dir), repo=REPO_URL,
system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))},
)
for path in measured.paths:
@@ -11187,7 +11181,7 @@ def _generate(args: argparse.Namespace) -> None:
)
write_release_advertised(Path(args.site_dir) / "src" / "generated")
write_if_changed(str(docs / "data.md"),
generate_data_page(exports, public / sync_measurements.SQLITE_PATH)
generate_data_page(exports, public / sitemeasurements.SQLITE_PATH)
+ reference_next())
PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
scripts/measurement_records.py
+149 -150
View File
@@ -1,50 +1,40 @@
"""Core probe records as the measurements repository stores them.
"""Core probe records as this repository stores them.
A run of core_probe.py produces one full record. It is stored as two parts:
A run of core_probe.py produces one full record, stored as one line:
records/<emulator>.jsonl one slim record per line, append-only
data/<emulator>/<id>.json.gz the bulky rest, named by the record id
measurements/<emulator>.jsonl one record per line, append-only
The slim record keeps what a reader needs to judge and reproduce the result:
claim, method, observable, result, statistics, host, inputs by hash, core
build and frontend commit, and per variant the values of its first
repetition. The sidecar holds every repetition in full (per-frame hashes,
core logs, declared options), the raw timing values the statistics were
computed from, and any compared value too large to sit in a line. Its
sha256 is in the record, so the line vouches for it. ``join`` puts the two
back into the full record, ``split`` is its inverse.
A line holds the whole run: claim, method, observable, result, host, inputs
by hash, core build and frontend commit, and every repetition of every
variant in full (per-frame hashes, core logs, declared options), with the
raw timing values the statistics were computed from.
A record id is the date and the first 12 hex digits of the sha256 of the
record without its id. A record carried over from the first layout keeps
the id it had, which hashes that full record.
record without its id, in canonical JSON. A record carried over from the
first layout keeps the id it had, which hashes that full record.
Standard library only: core_probe.py, sync_measurements.py and the site
Standard library only: core_probe.py, validate_schemas.py and the site
read it.
"""
from __future__ import annotations
import copy
import gzip
import hashlib
import json
import re
import subprocess
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
RECORD_VERSION = 2
MEASUREMENTS_REPO = "https://github.com/Abdess/retrobios-measurements"
RECORDS_DIR = "records"
DATA_DIR = "data"
# Fields a repetition keeps in the slim record, per variant, from the first one.
VARIANT_SUMMARY = ("observables", "files_opened", "files_probed_absent", "tracing")
# Observables too large for a line: their values stay in the sidecar.
BULKY_OBSERVABLES = ("frame_hashes",)
RECORD_VERSION = 3
RECORDS_DIR = "measurements"
ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
class RecordError(ValueError):
"""A record or sidecar that does not match its counterpart."""
"""A record line that cannot be read."""
def canonical(value: object) -> bytes:
@@ -55,6 +45,11 @@ def sha256_hex(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def value_sha1(value: object) -> str:
"""The sha1 of a value in canonical JSON."""
return hashlib.sha1(canonical(value)).hexdigest()
def slugify(text: str, limit: int = 40) -> str:
"""Lowercase words joined by hyphens, cut at a word boundary."""
words = re.findall(r"[a-z0-9]+", text.lower())
@@ -67,151 +62,55 @@ def slugify(text: str, limit: int = 40) -> str:
return out or "record"
def record_id(slim: dict) -> str:
body = {key: value for key, value in slim.items() if key != "id"}
return f"{slim['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
def record_id(record: dict) -> str:
body = {key: value for key, value in record.items() if key != "id"}
return f"{record['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
def line_text(slim: dict) -> str:
def seal(record: dict) -> dict:
"""The record with its id."""
return {**record, "id": record_id(record)}
def line_text(record: dict) -> str:
"""The record as one line of its .jsonl file, keys in a fixed order."""
return json.dumps(slim, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def sidecar_path(emulator: str, rid: str) -> str:
return f"{DATA_DIR}/{emulator}/{rid}.json.gz"
return json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def records_path(emulator: str) -> str:
return f"{RECORDS_DIR}/{emulator}.jsonl"
def sidecar_bytes(sidecar: dict) -> bytes:
"""gzip of the sidecar JSON, with no name and no time in the header."""
return gzip.compress(canonical(sidecar), compresslevel=9, mtime=0)
IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
def read_sidecar(blob: bytes) -> dict:
return json.loads(gzip.decompress(blob))
def _value_digest(value: object) -> str:
return hashlib.sha1(canonical(value)).hexdigest()
def split(full: dict) -> tuple[dict, dict]:
"""The full record as (slim record without id and data, sidecar)."""
slim = copy.deepcopy(full)
sidecar: dict = {"variants": [], "timing_raw": {}, "group_values": {}}
variants = []
for variant in slim["variants"]:
reps = variant["repetitions"]
sidecar["variants"].append({"label": variant["label"], "repetitions": reps})
first = reps[0]
summary = {key: first.get(key) for key in VARIANT_SUMMARY}
summary["observables"] = {
key: value for key, value in (first.get("observables") or {}).items()
if key not in BULKY_OBSERVABLES
}
variants.append({
"label": variant["label"],
"system_files": variant["system_files"],
"flags": variant["flags"],
"repetitions": len(reps),
"loaded": all(rep.get("loaded") for rep in reps),
**summary,
})
slim["variants"] = variants
for name, stats in (slim.get("timing") or {}).items():
sidecar["timing_raw"][name] = stats.pop("raw")
for group in ((slim.get("check") or {}).get("result") or {}).get("groups", []):
if isinstance(group.get("value"), (list, dict)):
digest = _value_digest(group["value"])
sidecar["group_values"][digest] = group.pop("value")
group["value_sha1"] = digest
return slim, sidecar
def join(slim: dict, sidecar: dict) -> dict:
"""The full record a slim record and its sidecar were split from."""
full = copy.deepcopy(slim)
full.pop("id", None)
full.pop("data", None)
labels = [v["label"] for v in full["variants"]]
if labels != [v["label"] for v in sidecar["variants"]]:
raise RecordError(f"variants differ: record {labels}, sidecar "
f"{[v['label'] for v in sidecar['variants']]}")
full["variants"] = [
{"label": variant["label"], "system_files": variant["system_files"],
"repetitions": copy.deepcopy(stored["repetitions"]), "flags": variant["flags"]}
for variant, stored in zip(full["variants"], sidecar["variants"])
]
for name, raw in sidecar.get("timing_raw", {}).items():
full["timing"][name]["raw"] = list(raw)
for group in ((full.get("check") or {}).get("result") or {}).get("groups", []):
digest = group.pop("value_sha1", None)
if digest is not None:
group["value"] = copy.deepcopy(sidecar["group_values"][digest])
return full
def seal(slim: dict, sidecar: dict, rid: str | None = None) -> tuple[dict, bytes]:
"""The slim record with its data reference and id, and the sidecar bytes.
``rid`` keeps an id given before (a record carried over from the first
layout), otherwise the id is computed.
"""
blob = sidecar_bytes(sidecar)
sealed = {**slim, "data": {"sha256": sha256_hex(blob), "bytes": len(blob)}}
sealed["id"] = rid or record_id(sealed)
return sealed, blob
def check_pair(slim: dict, blob: bytes) -> list[str]:
"""What is wrong between a stored line and its stored sidecar."""
problems = []
data = slim.get("data") or {}
if sha256_hex(blob) != data.get("sha256"):
problems.append(f"{slim.get('id')}: sidecar sha256 {sha256_hex(blob)} != {data.get('sha256')}")
if len(blob) != data.get("bytes"):
problems.append(f"{slim.get('id')}: sidecar is {len(blob)} bytes, record says {data.get('bytes')}")
if not problems:
try:
full = join(slim, read_sidecar(blob))
except (RecordError, KeyError, OSError, ValueError) as exc:
return [f"{slim.get('id')}: record and sidecar do not join: {exc}"]
if slim.get("migrated_from_version") == 1:
digest = sha256_hex(canonical(first_layout(full)))[:12]
if str(slim.get("id", "")).rsplit("-", 1)[-1] != digest:
problems.append(f"{slim.get('id')}: id does not hash the first-layout record ({digest})")
return problems
V2_IDENTITY = ("emulator", "subject", "supersedes", "migrated_from_version")
def first_layout(full: dict) -> dict:
"""A full record as the first layout wrote it: no emulator, subject or
def first_layout(record: dict) -> dict:
"""A record as the first layout wrote it: no id, emulator, subject or
supersedes, record_version 1."""
old = {key: value for key, value in full.items() if key not in V2_IDENTITY}
old = {key: value for key, value in record.items() if key not in IDENTITY}
old["record_version"] = 1
return old
def check_line(slim: dict) -> list[str]:
def check_line(record: dict) -> list[str]:
"""The structural rules a line holds, without a JSON Schema validator."""
rid = slim.get("id", "")
rid = record.get("id", "")
problems = []
if slim.get("record_version") != RECORD_VERSION:
problems.append(f"{rid}: record_version {slim.get('record_version')} != {RECORD_VERSION}")
if record.get("record_version") != RECORD_VERSION:
problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}")
if not ID_PATTERN.match(str(rid)):
problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>")
elif str(slim.get("recorded_at", ""))[:10] != rid[:10]:
problems.append(f"{rid}: id date differs from recorded_at {slim.get('recorded_at')}")
elif slim.get("migrated_from_version") is None and record_id(slim) != rid:
problems.append(f"{rid}: id does not hash the record ({record_id(slim)})")
elif str(record.get("recorded_at", ""))[:10] != rid[:10]:
problems.append(f"{rid}: id date differs from recorded_at {record.get('recorded_at')}")
elif record.get("migrated_from_version") == 1:
digest = sha256_hex(canonical(first_layout(record)))[:12]
if not rid.endswith(f"-{digest}"):
problems.append(f"{rid}: id does not hash the first-layout record ({digest})")
elif record_id(record) != rid:
problems.append(f"{rid}: id does not hash the record ({record_id(record)})")
for key in ("emulator", "subject"):
if not SLUG_PATTERN.match(str(slim.get(key, ""))):
problems.append(f"{rid}: {key} {slim.get(key)!r} is not a lowercase slug")
if not SLUG_PATTERN.match(str(record.get(key, ""))):
problems.append(f"{rid}: {key} {record.get(key)!r} is not a lowercase slug")
return problems
@@ -244,3 +143,103 @@ def live(records: Iterable[dict]) -> list[dict]:
records = list(records)
superseded = {r["supersedes"] for r in records if r.get("supersedes")}
return [r for r in records if r["id"] not in superseded]
class MeasurementsError(RuntimeError):
"""The records of a tree cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read from the tree."""
emulator: str
line: int
record: dict
def tree_files(root: Path) -> dict[str, Path]:
"""Repository path -> file of everything under ``root``/measurements."""
base = root / RECORDS_DIR
if not base.is_dir():
return {}
return {path.relative_to(root).as_posix(): path
for path in sorted(base.rglob("*")) if path.is_file()}
def read_records(root: Path) -> list[Stored]:
"""Every record line under ``root``, each checked against its rules."""
files = tree_files(root)
problems: list[str] = []
stored: list[Stored] = []
for path in sorted(files):
emulator = path[len(RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
continue
seen: set[str] = set()
try:
records = parse_lines(files[path].read_text(encoding="utf-8"), path)
except RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
stored.append(Stored(emulator, number, record))
if problems:
raise MeasurementsError("records under {}:\n {}".format(root, "\n ".join(problems)))
return stored
def _git(root: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
def committed_files(root: Path, ref: str) -> dict[str, bytes]:
"""Path -> content of every file under measurements/ at ``ref``."""
listing = _git(root, "ls-tree", "-r", "-z", "--name-only", ref, "--", RECORDS_DIR)
if listing.returncode != 0:
raise MeasurementsError(f"git ls-tree {ref}: {listing.stderr.decode(errors='replace').strip()}")
out = {}
for path in filter(None, listing.stdout.decode("utf-8").split("\0")):
blob = _git(root, "show", f"{ref}:{path}")
if blob.returncode != 0:
raise MeasurementsError(f"git show {ref}:{path}: {blob.stderr.decode(errors='replace').strip()}")
out[path] = blob.stdout
return out
def append_only_problems(root: Path, ref: str = "HEAD") -> list[str]:
"""What the tree under ``root`` changed or removed of the lines ``ref``
holds. Lines added at the end and new files are allowed."""
before, after = committed_files(root, ref), tree_files(root)
problems = []
for path, content in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
continue
now = after[path].read_bytes()
if now == content:
continue
if path.endswith(".jsonl"):
moved = appended_only(content.decode("utf-8"), now.decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
def last_commit(root: Path) -> str | None:
"""The last commit that touched the records: the one their permalinks name."""
proc = _git(root, "log", "-1", "--format=%H", "--", RECORDS_DIR)
commit = proc.stdout.decode().strip() if proc.returncode == 0 else ""
return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else None
scripts/refresh_stale.py
+1 -9
View File
@@ -27,9 +27,6 @@ Mapping (stale -> command):
--source mame --fetch
catalogs/fbneo recipes python -m scripts.scraper.romset_dat_importer
--source fbneo --fetch
measurements/records pin
python scripts/sync_measurements.py --fetch --bump,
which refuses a remote that rewrote a pinned line
A platform that inherits from a rescraped one (retropie and lakka from
retroarch) reads the parent's file: its jobs start once the parent's chain
@@ -66,7 +63,7 @@ CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs", "measurements")
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
@@ -410,11 +407,6 @@ def _command_for(
# redump / no-intro / tosec need manual DAT download.
return None
if area == "measurements":
# --fetch refuses a remote that rewrote a pinned line, and --bump
# only rewrites measurements-pin.json: nothing is committed.
return [sys.executable, "scripts/sync_measurements.py", "--fetch", "--bump"]
# coreinfo, ci, profiles: never auto-refresh.
return None
scripts/sitemeasurements.py
+369 -13
View File
@@ -1,24 +1,382 @@
"""The site's measurement pages, from the index sync_measurements.py writes.
"""The measurements the site shows, from the records under measurements/.
Three places read the same index: the Tested block of each emulator page,
the Measurements explorer page and the host table of the methodology page.
A number is never shown without its method and host beside it.
The records are core probe runs, one append-only .jsonl per emulator with one
full record per line (measurement_records.py). From them the build writes,
never committed:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
Three places read the index: the Tested block of each emulator page, the
Measurements explorer page and the host table of the methodology page. A
number is never shown without its method and host beside it. A record links
its line at the last commit that touched measurements/.
"""
from __future__ import annotations
import json
import os
import re
import sqlite3
import sys
from dataclasses import dataclass
from pathlib import Path
import measurement_records as mr
import sitemarkup as markup
from artifacts import json_text, write_if_changed
from siterender import _escape_tags
ROOT = Path(__file__).resolve().parent.parent
REPO = "https://github.com/Abdess/retrobios"
EXPLORER_PAGE = "measurements.md"
METHOD_PAGE = "wiki/measurements.md"
HOSTS_MARKER = "<!-- measurements-hosts -->"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "2"
# Observables the variants table leaves out: one value per sampled frame.
BULKY_OBSERVABLES = ("frame_hashes",)
INDEX_SCHEMA_VERSION = 1
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[mr.Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (source or {}).get("repo"),
"commit": (source or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, record_url TEXT NOT NULL,
line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (source or {}).get("repo") or "",
"measurements_commit": (source or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records, one per line of measurements/<emulator>.jsonl",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]),
permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]),
mr.value_sha1(group["value"]) if isinstance(group["value"], (list, dict)) else None,
json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
reps = variant["repetitions"]
first = reps[0]
# The first repetition stands for the variant, its per-frame hashes left out.
observables = {key: value for key, value in (first.get("observables") or {}).items()
if key not in BULKY_OBSERVABLES}
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
json_text(variant["flags"]), json_text(observables),
json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
json_text(first.get("tracing"))))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_commit(root: Path) -> str | None:
"""The commit the record links name, with a warning when the records on
disk are not the ones it holds."""
commit = mr.last_commit(root)
if commit is None:
print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links "
"lead nowhere until the records are committed.", file=sys.stderr)
return None
on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()}
if mr.committed_files(root, commit) != on_disk:
print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that "
"touched it: links to records added since lead nowhere until they are "
"committed.", file=sys.stderr)
return commit
def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators",
scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None,
repo: str = REPO, commit: str | None = None) -> Built:
"""Write the index and the databases under the site's public folder.
``commit`` is the revision the record links name, by default the last
commit that touched the records.
"""
stored = mr.read_records(root)
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
profiles = load_profiles(emulators_dir)
index = build_index(stored, source, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch)
paths.append(target)
return Built(index, paths)
# --------------------------------------------------------------------------
# Pages
# --------------------------------------------------------------------------
def by_emulator(index: dict) -> dict[str, list[dict]]:
out: dict[str, list[dict]] = {}
for item in index.get("items", []):
@@ -60,7 +418,7 @@ def tested_item(item: dict) -> list[str]:
+ (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "")
+ ".",
"",
f" [Full record]({item['record_url']}), [its data]({item['data_url']}), {runs}.",
f" [Full record]({item['record_url']}), {runs}.",
"",
]
return lines
@@ -91,7 +449,7 @@ def hosts_table(index: dict) -> list[str]:
computer = item["host"].split(", load ")[0]
hosts[computer] = hosts.get(computer, 0) + 1
if not hosts:
return ["No result is recorded at the pinned commit yet.", ""]
return ["No result is recorded yet.", ""]
return [
"| Computer | Tests |",
"|----------|------:|",
@@ -108,8 +466,8 @@ def with_hosts(page: str, index: dict) -> str:
def generate_explorer_page(index: dict) -> str:
commit = index.get("commit")
repo = index.get("repo") or ""
source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}"
if commit else "the measurements project, not read in this build")
source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
if commit else f"`{mr.RECORDS_DIR}/` of this repository")
return "\n".join([
"# Measurements",
"",
@@ -148,19 +506,17 @@ def data_page_section(path: Path) -> list[str]:
"## Measurements",
"",
"[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
"record at the pinned commit of [retrobios-measurements]"
"(https://github.com/Abdess/retrobios-measurements), and "
f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
"`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
"[`data/measurements/index.json`](data/measurements/index.json). A record line is "
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json) "
"and its data file against [`core-probe-data.schema.json`](api/v1/schemas/core-probe-data.schema.json).",
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
"",
"| Table | Columns |",
"|-------|---------|",
*(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
"",
"`meta` names the measurements commit and the record version. A list or object "
"`meta` names the commit the record links point to and the record version. A list or object "
"value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
"later one corrects.",
"",
scripts/sync_measurements.py
-564
View File
@@ -1,564 +0,0 @@
#!/usr/bin/env python3
"""Read the measurements repository at its pin and compile what the site shows.
The records live in their own repository (Abdess/retrobios-measurements),
one append-only .jsonl per emulator. This repository holds only the pin,
`measurements-pin.json`: the repository URL and the commit read. The clone
is kept at .cache/measurements/ and read at the pinned commit with git, so
its working tree may hold records not yet committed.
From the records at the pin it writes, never commits:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
`--fetch` clones or updates the clone, the only step that uses the network,
and refuses a remote whose history rewrote or removed a line the pin holds.
`--bump` then moves the pin to the fetched head.
"""
from __future__ import annotations
import argparse
import json
import os
import sqlite3
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement_records as mr # noqa: E402
from artifacts import json_text, write_if_changed, write_text_atomic # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PIN_FILE = ROOT / "measurements-pin.json"
CLONE = ROOT / ".cache" / "measurements"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "1"
INDEX_SCHEMA_VERSION = 1
class MeasurementsError(RuntimeError):
"""The measurements at the pin cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read at a commit."""
emulator: str
line: int
record: dict
# --------------------------------------------------------------------------
# Pin and git
# --------------------------------------------------------------------------
def load_pin(path: Path = PIN_FILE) -> dict:
pin = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(pin.get("repo"), str) or not isinstance(pin.get("commit"), str) \
or len(pin["commit"]) != 40:
raise MeasurementsError(f"{path}: needs a repo URL and a 40-character commit")
return pin
def write_pin(pin: dict, path: Path = PIN_FILE) -> None:
write_text_atomic(str(path), json.dumps(pin, indent=2, sort_keys=True) + "\n")
def git(clone: Path, *args: str, binary: bool = False) -> str | bytes:
proc = subprocess.run(["git", "-C", str(clone), *args], capture_output=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git {' '.join(args)}: {proc.stderr.decode(errors='replace').strip()}")
return proc.stdout if binary else proc.stdout.decode("utf-8")
def has_commit(clone: Path, commit: str) -> bool:
if not (clone / ".git").exists():
return False
proc = subprocess.run(["git", "-C", str(clone), "cat-file", "-e", f"{commit}^{{commit}}"],
capture_output=True, check=False)
return proc.returncode == 0
def tree_files(clone: Path, commit: str) -> dict[str, str]:
"""Path -> blob id of every file under records/ and data/ at a commit."""
out: dict[str, str] = {}
listing = git(clone, "ls-tree", "-r", commit, "--", mr.RECORDS_DIR, mr.DATA_DIR)
for row in str(listing).splitlines():
meta, _, path = row.partition("\t")
out[path] = meta.split()[2]
return out
def show(clone: Path, commit: str, path: str) -> bytes:
return bytes(git(clone, "show", f"{commit}:{path}", binary=True))
# --------------------------------------------------------------------------
# Reading and checking
# --------------------------------------------------------------------------
def read_records(clone: Path, commit: str) -> list[Stored]:
"""Every record line at a commit, each checked against its rules and data."""
files = tree_files(clone, commit)
problems: list[str] = []
stored: list[Stored] = []
for path in sorted(p for p in files if p.startswith(f"{mr.RECORDS_DIR}/")):
emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: records/ holds <emulator>.jsonl files only")
continue
seen: set[str] = set()
try:
records = mr.parse_lines(show(clone, commit, path).decode("utf-8"), path)
except mr.RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in mr.check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
data = mr.sidecar_path(emulator, str(record.get("id")))
if data not in files:
problems.append(f"{where}: {data} is missing")
else:
problems.extend(f"{where}: {p}" for p in mr.check_pair(record, show(clone, commit, data)))
stored.append(Stored(emulator, number, record))
if problems:
raise MeasurementsError("records at {}:\n {}".format(commit[:12], "\n ".join(problems)))
return stored
def append_only_problems(clone: Path, old: str, new: str) -> list[str]:
"""What ``new`` changed or removed of the lines and data ``old`` holds."""
before, after = tree_files(clone, old), tree_files(clone, new)
problems = []
for path, blob in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
elif after[path] == blob:
continue
elif path.endswith(".jsonl"):
moved = mr.appended_only(show(clone, old, path).decode("utf-8"),
show(clone, new, path).decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: Stored, runs: int, pin: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(pin["repo"], pin["commit"], mr.records_path(entry.emulator), entry.line),
"data_url": permalink(pin["repo"], pin["commit"], mr.sidecar_path(entry.emulator, record["id"])),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[Stored], pin: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), pin or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (pin or {}).get("repo"),
"commit": (pin or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL,
data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[Stored], pin: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (pin or {}).get("repo") or "",
"measurements_commit": (pin or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records: the record lines, not their data files",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, pin or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: Stored, pin: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"],
permalink(pin.get("repo", ""), pin.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]) if "value" in group else None,
group.get("value_sha1"), json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]),
json_text(variant["flags"]), json_text(variant["observables"]),
json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]),
json_text(variant["tracing"])))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_at_pin(clone: Path, pin: dict, require: bool) -> list[Stored] | None:
"""The records at the pin, or None when the clone does not hold it."""
if has_commit(clone, pin["commit"]):
return read_records(clone, pin["commit"])
message = (f"measurements commit {pin['commit'][:12]} is not in {clone}: run "
f"python scripts/sync_measurements.py --fetch")
if require:
raise MeasurementsError(message)
print(f"WARNING: {message}. The site lists no measurements.", file=sys.stderr)
return None
def build(public: Path, clone: Path = CLONE, pin_path: Path = PIN_FILE,
emulators_dir: Path = ROOT / "emulators", require: bool = False,
scratch: Path = ROOT / "tmp" / "site",
system_names: dict[str, str] | None = None) -> Built:
"""Write the index and the databases under the site's public folder."""
pin = load_pin(pin_path)
stored = records_at_pin(clone, pin, require)
profiles = load_profiles(emulators_dir)
if stored is None:
pin = None
index = build_index(stored or [], pin, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored or [], pin, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored or []}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored or [] if s.emulator == emulator], pin, profiles, scratch)
paths.append(target)
return Built(index, paths)
def fetch(clone: Path, pin: dict) -> str:
"""Clone or update the measurements repository. Returns the remote head."""
if not (clone / ".git").exists():
clone.parent.mkdir(parents=True, exist_ok=True)
proc = subprocess.run(["git", "clone", "--quiet", pin["repo"], str(clone)],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git clone {pin['repo']}: {proc.stderr.strip()}")
else:
git(clone, "fetch", "--quiet", "origin")
head = str(git(clone, "rev-parse", "refs/remotes/origin/HEAD")).strip()
branch = str(git(clone, "rev-parse", "--abbrev-ref", "HEAD")).strip()
if branch != "HEAD":
proc = subprocess.run(["git", "-C", str(clone), "merge", "--ff-only", "--quiet", head],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
print(f"WARNING: {clone} has local commits, its branch was left as is: "
f"{proc.stderr.strip()}", file=sys.stderr)
return head
def remote_head(repo: str) -> str:
proc = subprocess.run(["git", "ls-remote", repo, "HEAD"], capture_output=True, text=True,
check=False, timeout=60)
if proc.returncode != 0 or not proc.stdout.strip():
raise MeasurementsError(f"git ls-remote {repo}: {proc.stderr.strip() or 'no HEAD'}")
return proc.stdout.split()[0]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument("--fetch", action="store_true",
help="clone or update the clone, and check the remote kept every line the pin holds")
parser.add_argument("--bump", action="store_true",
help=f"after --fetch, move {PIN_FILE.name} to the fetched head")
parser.add_argument("--check-only", action="store_true",
help="check the records at the pin and write nothing")
parser.add_argument("--clone", default=str(CLONE), help="the clone (default: %(default)s)")
parser.add_argument("--pin", default=str(PIN_FILE), help="the pin file (default: %(default)s)")
parser.add_argument("--site-dir", default=str(ROOT / "website"),
help="the site project whose public/ folder is written (default: %(default)s)")
args = parser.parse_args(argv)
if args.bump and not args.fetch:
parser.error("--bump needs --fetch")
clone, pin_path = Path(args.clone), Path(args.pin)
try:
pin = load_pin(pin_path)
if args.fetch:
head = fetch(clone, pin)
problems = append_only_problems(clone, pin["commit"], head) \
if has_commit(clone, pin["commit"]) else [f"pin {pin['commit']} is not in the remote history"]
if problems:
print("ERROR: the remote changed records the pin holds:\n " + "\n ".join(problems),
file=sys.stderr)
return 1
records = read_records(clone, head)
pinned = len(read_records(clone, pin["commit"]))
print(f"{head[:12]}: {len(records)} records, {len(records) - pinned} after the pin "
f"{pin['commit'][:12]}")
if args.bump and head != pin["commit"]:
write_pin({**pin, "commit": head}, pin_path)
print(f"pin moved to {head}")
return 0
if args.check_only:
stored = records_at_pin(clone, pin, require=True)
print(f"{pin['commit'][:12]}: {len(stored or [])} records, every line and data file checked")
return 0
built = build(Path(args.site_dir) / "public", clone, pin_path, require=True)
print(f"{built.index['records']} records, {built.index['count']} subjects: "
+ ", ".join(str(p.relative_to(Path(args.site_dir))) for p in built.paths))
return 0
except MeasurementsError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
from common import run_cli
run_cli(main)
scripts/validate_schemas.py
+14 -31
View File
@@ -593,46 +593,29 @@ def _format_errors() -> list[str]:
]
def _measurement_errors(clone: Path | None = None, pin_path: Path | None = None) -> list[str]:
"""Every record line and data file at the measurements pin, against their
schemas and the rules sync_measurements.py applies.
The records live in their own repository. Without a clone holding the
pin there is nothing to read, and the run says so.
"""
import sync_measurements # noqa: PLC0415
def _measurement_errors(root: Path | None = None) -> list[str]:
"""Every record line under measurements/, against its schema and the
rules measurement_records.read_records applies."""
import measurement_records as mr # noqa: PLC0415
clone = clone or sync_measurements.CLONE
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
if not sync_measurements.has_commit(clone, pin["commit"]):
print(f"SKIP measurements: {pin['commit'][:12]} is not in {clone} "
"(python scripts/sync_measurements.py --fetch)")
return []
commit = pin["commit"]
record_validator = _validator("core-probe-record.schema.json")
data_validator = _validator("core-probe-data.schema.json")
root = root or ROOT
validator = _validator("core-probe-record.schema.json")
out: list[str] = []
files = sync_measurements.tree_files(clone, commit)
for path in sorted(p for p in files if p.endswith(".jsonl")):
text = sync_measurements.show(clone, commit, path).decode("utf-8")
for number, line in enumerate(text.splitlines(), 1):
for path, file in sorted(mr.tree_files(root).items()):
if not path.endswith(".jsonl"):
continue
for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), 1):
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
out.append(f"measurements {path}:{number}: {exc}")
out.append(f"{path}:{number}: {exc}")
continue
out.extend(_errors(record_validator, record, f"measurements {path}:{number}"))
data = mr.sidecar_path(path[len("records/"):-len(".jsonl")], str(record.get("id")))
if data in files:
out.extend(_errors(data_validator,
mr.read_sidecar(sync_measurements.show(clone, commit, data)),
f"measurements {data}"))
out.extend(_errors(validator, record, f"{path}:{number}"))
if not out:
try:
sync_measurements.read_records(clone, commit)
except sync_measurements.MeasurementsError as exc:
out.append(f"measurements {exc}")
mr.read_records(root)
except mr.MeasurementsError as exc:
out.append(str(exc))
return out
tests/test_core_probe.py
+13 -27
View File
@@ -261,34 +261,25 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual(record["variants"][3]["repetitions"][0]["files_probed_absent"], ["card.bin"])
self.assertEqual(first["frames"], 3)
self.assertEqual(record["emulator"], "fake")
self.assertEqual(sorted(p.relative_to(record_dir).as_posix()
for p in record_dir.rglob("*") if p.is_file()),
sorted(["records/fake.jsonl", f"data/fake/{self._only_id(record_dir)}.json.gz"]))
line = json.loads((record_dir / "records" / "fake.jsonl").read_text())
blob = (record_dir / "data" / "fake" / f"{line['id']}.json.gz").read_bytes()
self.assertEqual([p.relative_to(record_dir).as_posix()
for p in record_dir.rglob("*") if p.is_file()], ["measurements/fake.jsonl"])
line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text())
self.assertEqual(measurement_records.check_line(line), [])
self.assertEqual(measurement_records.check_pair(line, blob), [])
# The line and its sidecar give back the record the run printed.
self.assertEqual(measurement_records.join(line, measurement_records.read_sidecar(blob)), record)
# The line is the record the run printed, with its id.
self.assertEqual({k: v for k, v in line.items() if k != "id"}, record)
try:
from jsonschema import Draft202012Validator
except ImportError:
return
for name, document in (("core-probe-record", line),
("core-probe-data", measurement_records.read_sidecar(blob))):
schema = json.loads((REPO_ROOT / "schemas" / f"{name}.schema.json").read_text())
self.assertEqual(list(Draft202012Validator(schema).iter_errors(document)), [], name)
@staticmethod
def _only_id(record_dir: Path) -> str:
return json.loads((record_dir / "records" / "fake.jsonl").read_text())["id"]
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [])
def test_second_run_is_appended_beside_the_first(self) -> None:
record_dir = self.tmp / "appended"
argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000",
"--record", str(record_dir), "--subject", "smoke"]
self.assertEqual(self.probe(argv)[0], 0)
lines = record_dir / "records" / "fake.jsonl"
lines = record_dir / "measurements" / "fake.jsonl"
first = lines.read_text()
code, record = self.probe(argv)
self.assertEqual(code, 0)
@@ -301,16 +292,11 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual(code, 2)
self.assertEqual(len(lines.read_text().splitlines()), 3)
def test_recording_inside_the_tracked_tree_is_refused(self) -> None:
inside = REPO_ROOT / "measurements"
err = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
code = core_probe.main(["run", "--core", str(self.core), "--frames", "1", "--no-trace",
"--record", str(inside), "--work-dir", str(self.tmp / "work")])
self.assertEqual(code, 2)
self.assertIn("inside the tracked tree", err.getvalue())
self.assertIn(".cache/measurements", err.getvalue())
self.assertFalse(inside.exists())
def test_record_defaults_to_this_checkout(self) -> None:
parser = core_probe.build_parser()
args = parser.parse_args(["run", "--core", sys.executable, "--record", "--frames", "1"])
self.assertEqual(Path(args.record), REPO_ROOT)
self.assertIsNone(parser.parse_args(["run", "--core", sys.executable]).record)
def test_bench_stats(self) -> None:
code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1",
tests/test_measurements.py
+177 -172
View File
@@ -1,16 +1,14 @@
"""Core probe records in their own repository, and what the site makes of them.
"""Core probe records under measurements/, and what the site makes of them.
A synthetic measurements repository is built with git under tmp/tests:
records split into a line and a sidecar, read at a pinned commit, compiled
into the index and the SQLite files. The append-only rule is checked the
way --fetch applies it, and two branches that each add a line merge.
Synthetic record trees are built under tmp/tests: one full record per line,
read from the tree, compiled into the index and the SQLite files. The append-only rule is checked against a commit, two branches that
each add a line merge, and the records this repository holds are checked
against HEAD.
"""
from __future__ import annotations
import contextlib
import copy
import io
import json
import os
import re
@@ -26,10 +24,10 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import measurement_records as mr # noqa: E402
import sitemeasurements # noqa: E402
import sync_measurements as sm # noqa: E402
import sitemeasurements as sm # noqa: E402
REPO_URL = "https://github.com/Abdess/retrobios-measurements"
REPO_URL = "https://github.com/Abdess/retrobios"
COMMIT = "e" * 40
SHA1 = "a" * 40
SHA256 = "b" * 64
@@ -95,6 +93,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
return record
def git(cwd: Path, *args: str) -> str:
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
"GIT_COMMITTER_EMAIL": "t@t", "GIT_CEILING_DIRECTORIES": str(cwd.parent)}
@@ -103,11 +103,7 @@ def git(cwd: Path, *args: str) -> str:
def add_record(tree: Path, full: dict) -> dict:
slim, sidecar = mr.split(full)
sealed, blob = mr.seal(slim, sidecar)
data = tree / mr.sidecar_path(sealed["emulator"], sealed["id"])
data.parent.mkdir(parents=True, exist_ok=True)
data.write_bytes(blob)
sealed = mr.seal(full)
lines = tree / mr.records_path(sealed["emulator"])
lines.parent.mkdir(parents=True, exist_ok=True)
with lines.open("a", encoding="utf-8") as handle:
@@ -122,128 +118,126 @@ def commit_all(tree: Path, message: str) -> str:
class Workspace(unittest.TestCase):
"""A measurements repository with three records and a pin on it."""
"""A tree with three records under measurements/ and one profile."""
def setUp(self) -> None:
(REPO_ROOT / "tmp" / "tests").mkdir(parents=True, exist_ok=True)
self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
self.addCleanup(shutil.rmtree, self.tmp, True)
self.repo = self.tmp / "remote"
self.repo.mkdir()
git(self.repo, "init", "-q", "-b", "main")
(self.repo / ".gitattributes").write_text("records/*.jsonl merge=union\n")
self.first = add_record(self.repo, full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
self.bench = add_record(self.repo, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00",
self.tree = self.tmp / "tree"
self.tree.mkdir()
self.first = add_record(self.tree, full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
self.bench = add_record(self.tree, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00",
kind="bench"))
self.beta = add_record(self.repo, full_record("beta", "card", "2026-10-02T10:00:00+00:00",
self.beta = add_record(self.tree, full_record("beta", "card", "2026-10-02T10:00:00+00:00",
values=("1" * 40, "1" * 40)))
self.commit = commit_all(self.repo, "feat: first records")
self.pin = self.tmp / "pin.json"
sm.write_pin({"repo": str(self.repo), "commit": self.commit}, self.pin)
self.emulators = self.tmp / "emulators"
self.emulators.mkdir()
(self.emulators / "alpha.yml").write_text(
'emulator: "Alpha"\ntype: libretro\nsystems:\n - sega-saturn\nfiles: []\n')
def built(self, public: Path, clone: Path | None = None) -> sm.Built:
return sm.build(public, clone or self.repo, self.pin, self.emulators, require=True,
scratch=self.tmp / "scratch", system_names={"sega-saturn": "Sega Saturn"})
def built(self, public: Path) -> sm.Built:
return sm.build(public, self.tree, self.emulators, scratch=self.tmp / "scratch",
system_names={"sega-saturn": "Sega Saturn"}, repo=REPO_URL, commit=COMMIT)
def repository(self) -> str:
"""Make the tree a git repository, its records committed on main."""
git(self.tree, "init", "-q", "-b", "main")
(self.tree / ".gitattributes").write_text("measurements/*.jsonl merge=union\n")
return commit_all(self.tree, "feat: first records")
class RecordSplit(unittest.TestCase):
def test_round_trip(self) -> None:
class RecordLine(unittest.TestCase):
def test_a_line_holds_the_whole_record(self) -> None:
for kind in ("check", "bench"):
full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00", kind=kind)
slim, sidecar = mr.split(full)
sealed, blob = mr.seal(slim, sidecar)
sealed = mr.seal(full)
with self.subTest(kind=kind):
self.assertEqual(mr.join(sealed, mr.read_sidecar(blob)), full)
self.assertEqual({k: v for k, v in sealed.items() if k != "id"}, full)
self.assertEqual(json.loads(mr.line_text(sealed)), sealed)
self.assertEqual(mr.check_line(sealed), [])
self.assertEqual(mr.check_pair(sealed, blob), [])
self.assertNotIn("logs", json.dumps(sealed["variants"]))
self.assertNotIn("frame_hashes", json.dumps(sealed["variants"]))
self.assertIn("frame_hashes", mr.line_text(sealed))
self.assertIn("card present", mr.line_text(sealed))
self.assertNotIn("\n", mr.line_text(sealed))
def test_bulky_compared_value_moves_to_the_sidecar(self) -> None:
full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00", observable="frame_hashes")
slim, sidecar = mr.split(full)
groups = slim["check"]["result"]["groups"]
self.assertTrue(all("value" not in g and len(g["value_sha1"]) == 40 for g in groups))
sealed, blob = mr.seal(slim, sidecar)
self.assertEqual(mr.join(sealed, mr.read_sidecar(blob)), full)
def test_sidecar_bytes_and_id_are_reproducible(self) -> None:
def test_id_is_reproducible(self) -> None:
full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00")
one = mr.seal(*mr.split(full))
two = mr.seal(*mr.split(copy.deepcopy(full)))
self.assertEqual(one, two)
self.assertRegex(one[0]["id"], r"^2026-10-01-[0-9a-f]{12}$")
self.assertEqual(mr.seal(full), mr.seal(copy.deepcopy(full)))
self.assertRegex(mr.seal(full)["id"], r"^2026-10-01-[0-9a-f]{12}$")
def test_tampering_is_named(self) -> None:
sealed, blob = mr.seal(*mr.split(full_record("alpha", "card", "2026-10-01T10:00:00+00:00")))
self.assertTrue(mr.check_pair(sealed, blob + b"x"))
changed = {**sealed, "subject": "other"}
self.assertTrue(any("does not hash" in p for p in mr.check_line(changed)))
def test_a_campaign_line_stays_under_20_kb(self) -> None:
"""Eleven variants, two repetitions each, long logs: the line is what
a reader opens, the logs are in the sidecar."""
full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00",
values=tuple(f"{i:040x}" for i in range(11)))
for variant in full["variants"]:
for rep in variant["repetitions"]:
rep["logs"] = rep["logs"] * 200
rep["observables"]["frame_hashes"] = [[k, SHA1] for k in range(0, 3000, 150)]
sealed, _blob = mr.seal(*mr.split(full))
self.assertLess(len(mr.line_text(sealed).encode()), 20_000)
sealed = mr.seal(full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
for changed in ({**sealed, "subject": "other"},
{**sealed, "variants": sealed["variants"][:1]}):
self.assertTrue(any("does not hash" in p for p in mr.check_line(changed)))
def test_first_layout_record_keeps_its_id(self) -> None:
old = full_record("alpha", "card", "2026-10-01T10:00:00+00:00")
old = mr.first_layout(old)
old = mr.first_layout(full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
rid = f"2026-10-01-{mr.sha256_hex(mr.canonical(old))[:12]}"
full = {**old, "record_version": 2, "emulator": "alpha", "subject": "card",
"supersedes": None, "migrated_from_version": 1}
sealed, blob = mr.seal(*mr.split(full), rid)
self.assertEqual(sealed["id"], rid)
self.assertEqual(mr.check_line(sealed), [])
self.assertEqual(mr.check_pair(sealed, blob), [])
self.assertEqual(mr.first_layout(mr.join(sealed, mr.read_sidecar(blob))), old)
forged = {**sealed, "id": "2026-10-01-000000000000"}
self.assertTrue(mr.check_pair(forged, blob))
line = {**old, "record_version": mr.RECORD_VERSION, "emulator": "alpha", "subject": "card",
"supersedes": None, "migrated_from_version": 1, "id": rid}
self.assertEqual(mr.check_line(line), [])
self.assertEqual(mr.first_layout(line), old)
self.assertTrue(mr.check_line({**line, "id": "2026-10-01-000000000000"}))
changed = copy.deepcopy(line)
changed["variants"][0]["repetitions"][0]["logs"] = []
self.assertTrue(any("first-layout" in p for p in mr.check_line(changed)))
class SyncAndIndex(Workspace):
def test_records_are_read_at_the_pin_not_the_working_tree(self) -> None:
add_record(self.repo, full_record("alpha", "card", "2026-10-03T10:00:00+00:00"))
records = sm.read_records(self.repo, self.commit)
self.assertEqual(len(records), 3)
class ReadAndIndex(Workspace):
def test_records_are_read_from_the_tree(self) -> None:
records = mr.read_records(self.tree)
self.assertEqual([(s.emulator, s.line) for s in records], [("alpha", 1), ("alpha", 2), ("beta", 1)])
self.assertEqual(mr.read_records(self.tmp / "empty"), [])
def test_a_broken_tree_is_named(self) -> None:
stray = self.tree / mr.RECORDS_DIR / "data" / "beta" / "2026-10-02-000000000000.json.gz"
stray.parent.mkdir(parents=True)
stray.write_bytes(b"x")
lines = self.tree / mr.records_path("beta")
lines.write_text(lines.read_text() * 2)
with self.assertRaises(mr.MeasurementsError) as caught:
mr.read_records(self.tree)
message = str(caught.exception)
self.assertIn(f"{mr.RECORDS_DIR}/data/beta/2026-10-02-000000000000.json.gz: "
f"{mr.RECORDS_DIR}/ holds <emulator>.jsonl files only", message)
self.assertIn(f"measurements/beta.jsonl:2: id {self.beta['id']} appears twice", message)
def test_index_shape(self) -> None:
index = self.built(self.tmp / "public").index
self.assertEqual({k: index[k] for k in ("schema_version", "kind", "repo", "commit", "records", "count")},
{"schema_version": 1, "kind": "measurements", "repo": str(self.repo),
"commit": self.commit, "records": 3, "count": 3})
{"schema_version": 1, "kind": "measurements", "repo": REPO_URL,
"commit": COMMIT, "records": 3, "count": 3})
keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind",
"claim", "method", "observable", "variable", "status", "statement", "content",
"variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id",
"runs", "record_url", "data_url", "sqlite"}
"runs", "record_url", "sqlite"}
for item in index["items"]:
self.assertEqual(set(item), keys)
alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"}
self.assertEqual(alpha["card"]["status"], "differs")
self.assertEqual(alpha["card"]["system_names"], ["Sega Saturn"])
self.assertEqual(alpha["card"]["record_url"],
f"{self.repo}/blob/{self.commit}/records/alpha.jsonl#L1")
f"{REPO_URL}/blob/{COMMIT}/measurements/alpha.jsonl#L1")
self.assertEqual(alpha["speed"]["status"], "measured")
self.assertIn("ms per frame", alpha["speed"]["statement"])
self.assertIn("Test CPU", alpha["speed"]["host"])
written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text())
self.assertEqual(written, index)
def test_links_name_the_last_commit_that_touched_the_records(self) -> None:
first = self.repository()
(self.tree / "other.txt").write_text("x")
commit_all(self.tree, "chore: unrelated")
self.assertEqual(mr.last_commit(self.tree), first)
index = sm.build(self.tmp / "public", self.tree, self.emulators, scratch=self.tmp / "scratch").index
self.assertEqual(index["commit"], first)
self.assertTrue(all(f"/blob/{first}/measurements/" in i["record_url"] for i in index["items"]))
def test_latest_record_of_a_subject_wins_and_a_superseded_one_is_dropped(self) -> None:
later = full_record("alpha", "card", "2026-10-05T10:00:00+00:00", values=("3" * 40, "3" * 40))
later["supersedes"] = self.first["id"]
add_record(self.repo, later)
sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "feat: rerun")}, self.pin)
add_record(self.tree, later)
card = next(i for i in self.built(self.tmp / "public").index["items"]
if i["emulator"] == "alpha" and i["subject"] == "card")
self.assertEqual((card["status"], card["runs"]), ("same", 1))
@@ -257,7 +251,7 @@ class SyncAndIndex(Workspace):
"SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
self.assertEqual(tuple(tables), sm.TABLES)
count = lambda table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: E731
records = [s.record for s in sm.read_records(self.repo, self.commit)]
records = [s.record for s in mr.read_records(self.tree)]
self.assertEqual(count("runs"), len(records))
self.assertEqual(count("checks"), sum(1 for r in records if r.get("check")))
self.assertEqual(count("benchmarks"), 2 * sum(1 for r in records if r.get("timing")))
@@ -266,10 +260,12 @@ class SyncAndIndex(Workspace):
sum(len(r["check"]["result"]["groups"]) for r in records if r.get("check")))
self.assertEqual(count("hosts"), 1)
self.assertEqual(count("cores"), 1)
self.assertEqual(dict(connection.execute("SELECT key, value FROM meta"))["measurements_commit"],
self.commit)
meta = dict(connection.execute("SELECT key, value FROM meta"))
self.assertEqual((meta["measurements_repo"], meta["measurements_commit"]), (REPO_URL, COMMIT))
self.assertEqual(connection.execute("SELECT system FROM emulator_systems").fetchall(),
[("sega-saturn",)])
self.assertEqual(connection.execute("SELECT record_url FROM runs WHERE line = 1 AND emulator = 'beta'")
.fetchone()[0], f"{REPO_URL}/blob/{COMMIT}/measurements/beta.jsonl#L1")
indexes = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")}
self.assertTrue({"runs_emulator_idx", "emulator_systems_system_idx", "checks_claim_idx"} <= indexes)
connection.close()
@@ -286,15 +282,11 @@ class SyncAndIndex(Workspace):
for path in first.paths:
self.assertEqual(path.read_bytes(), (two / path.relative_to(one)).read_bytes(), path.name)
def test_a_missing_pin_fails_only_when_required(self) -> None:
sm.write_pin({"repo": str(self.repo), "commit": "f" * 40}, self.pin)
with self.assertRaises(sm.MeasurementsError):
self.built(self.tmp / "public")
with contextlib.redirect_stderr(io.StringIO()) as err:
built = sm.build(self.tmp / "public", self.repo, self.pin, self.emulators, require=False,
scratch=self.tmp / "scratch")
self.assertIn("WARNING", err.getvalue())
self.assertEqual((built.index["count"], built.index["commit"]), (0, None))
def test_no_records_writes_an_empty_index(self) -> None:
shutil.rmtree(self.tree / mr.RECORDS_DIR)
built = self.built(self.tmp / "public")
self.assertEqual((built.index["records"], built.index["count"]), (0, 0))
self.assertEqual(len(built.paths), 2)
def test_every_line_validates_against_the_schema(self) -> None:
try:
@@ -303,68 +295,75 @@ class SyncAndIndex(Workspace):
self.skipTest("jsonschema is not installed")
import validate_schemas # noqa: PLC0415
self.assertEqual(validate_schemas._measurement_errors(self.repo, self.pin), [])
lines = self.repo / mr.records_path("beta")
self.assertEqual(validate_schemas._measurement_errors(self.tree), [])
lines = self.tree / mr.records_path("beta")
record = json.loads(lines.read_text())
record["kind"] = "guess"
lines.write_text(mr.line_text(record) + "\n")
sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "bad")}, self.pin)
errors = validate_schemas._measurement_errors(self.repo, self.pin)
self.assertTrue(any("records/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
errors = validate_schemas._measurement_errors(self.tree)
self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
self.assertIsNotNone(Draft202012Validator)
class AppendOnly(Workspace):
def fetch(self, *extra: str) -> tuple[int, str]:
err = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
code = sm.main(["--fetch", "--clone", str(self.tmp / "clone"), "--pin", str(self.pin), *extra])
return code, err.getvalue()
def setUp(self) -> None:
super().setUp()
self.commit = self.repository()
def test_an_appended_line_moves_the_pin(self) -> None:
self.assertEqual(self.fetch()[0], 0)
add_record(self.repo, full_record("alpha", "card", "2026-10-04T10:00:00+00:00"))
head = commit_all(self.repo, "feat: rerun")
self.assertEqual(self.fetch("--bump")[0], 0)
self.assertEqual(sm.load_pin(self.pin)["commit"], head)
def test_an_appended_line_and_a_new_file_pass(self) -> None:
add_record(self.tree, full_record("alpha", "card", "2026-10-04T10:00:00+00:00"))
add_record(self.tree, full_record("gamma", "card", "2026-10-04T10:00:00+00:00"))
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"), [])
def test_a_changed_line_is_refused(self) -> None:
lines = self.repo / mr.records_path("alpha")
def test_a_changed_line_is_named(self) -> None:
lines = self.tree / mr.records_path("alpha")
text = lines.read_text().splitlines()
text[0] = text[0].replace('"Each card, twice."', '"Each card, once."')
lines.write_text("\n".join(text) + "\n")
commit_all(self.repo, "rewrite")
code, err = self.fetch("--bump")
self.assertEqual(code, 1)
self.assertIn("records/alpha.jsonl: line 1 changed", err)
self.assertEqual(sm.load_pin(self.pin)["commit"], self.commit)
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"),
["measurements/alpha.jsonl: line 1 changed"])
def test_a_removed_line_or_data_file_is_refused(self) -> None:
lines = self.repo / mr.records_path("alpha")
def test_a_removed_line_or_file_is_named(self) -> None:
lines = self.tree / mr.records_path("alpha")
lines.write_text(lines.read_text().splitlines()[0] + "\n")
(self.repo / mr.sidecar_path("beta", self.beta["id"])).unlink()
commit_all(self.repo, "remove")
code, err = self.fetch()
self.assertEqual(code, 1)
self.assertIn("records/alpha.jsonl: line 2 removed", err)
self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", err)
(self.tree / mr.records_path("beta")).unlink()
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"),
["measurements/alpha.jsonl: line 2 removed", "measurements/beta.jsonl: removed"])
def test_two_branches_adding_a_line_merge_cleanly(self) -> None:
git(self.repo, "checkout", "-q", "-b", "left")
add_record(self.repo, full_record("alpha", "card", "2026-10-06T10:00:00+00:00"))
commit_all(self.repo, "left")
git(self.repo, "checkout", "-q", "main")
git(self.repo, "checkout", "-q", "-b", "right")
add_record(self.repo, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench",
git(self.tree, "checkout", "-q", "-b", "left")
add_record(self.tree, full_record("alpha", "card", "2026-10-06T10:00:00+00:00"))
commit_all(self.tree, "left")
git(self.tree, "checkout", "-q", "main")
git(self.tree, "checkout", "-q", "-b", "right")
add_record(self.tree, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench",
values=("4" * 40,)))
commit_all(self.repo, "right")
git(self.repo, "merge", "-q", "--no-edit", "left")
merged = git(self.repo, "rev-parse", "HEAD")
self.assertEqual(sm.append_only_problems(self.repo, self.commit, merged), [])
records = sm.read_records(self.repo, merged)
commit_all(self.tree, "right")
git(self.tree, "merge", "-q", "--no-edit", "left")
self.assertEqual(mr.append_only_problems(self.tree, self.commit), [])
records = mr.read_records(self.tree)
self.assertEqual(sum(1 for s in records if s.emulator == "alpha"), 4)
class CommittedRecords(unittest.TestCase):
"""The records this repository holds."""
def test_no_line_changed_or_vanished_since_head(self) -> None:
if subprocess.run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "-q", "HEAD"],
capture_output=True, check=False).returncode != 0:
self.skipTest("not a git checkout")
self.assertEqual(mr.append_only_problems(REPO_ROOT, "HEAD"), [])
def test_every_record_reads_and_joins_its_data(self) -> None:
records = mr.read_records(REPO_ROOT)
self.assertGreaterEqual(len(records), 7)
def test_jsonl_files_merge_by_union(self) -> None:
rules = (REPO_ROOT / ".gitattributes").read_text().splitlines()
rule = next(line for line in rules if line.startswith(f"{mr.RECORDS_DIR}/*.jsonl "))
self.assertIn("merge=union", rule.split())
class SitePages(unittest.TestCase):
ITEM = {
"emulator": "beetle_saturn", "emulator_name": "Beetle Saturn", "profiled": True,
@@ -376,26 +375,25 @@ class SitePages(unittest.TestCase):
"host": "Test CPU, 12 CPUs", "core": {"library": "Beetle Saturn v1.32.1", "commit": "2ce77e2" * 5 + "abcde",
"file": "x.so", "sha256": SHA256, "repo": None, "build": "make"},
"frontend_commit": None, "id": "2026-10-27-43527b763ace", "runs": 2,
"record_url": f"{REPO_URL}/blob/{'e' * 40}/records/beetle_saturn.jsonl#L4",
"data_url": f"{REPO_URL}/blob/{'e' * 40}/data/beetle_saturn/2026-10-27-43527b763ace.json.gz",
"record_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_saturn.jsonl#L4",
"sqlite": "downloads/measurements/beetle_saturn.sqlite",
}
def test_tested_block(self) -> None:
self.assertEqual(sitemeasurements.tested_section("beetle_saturn", []), [])
text = "\n".join(sitemeasurements.tested_section("beetle_saturn", [self.ITEM]))
self.assertEqual(sm.tested_section("beetle_saturn", []), [])
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM]))
self.assertTrue(text.startswith("## Tested\n"))
for expected in ("**Result:** `same`", "**Method:** Twice per card. Every input fixed.", "**Ran on:** Test CPU, 12 CPUs",
self.ITEM["record_url"], self.ITEM["data_url"], "1 flag raised", "2 runs",
self.ITEM["record_url"], "1 flag raised", "2 runs",
"../downloads/measurements/beetle_saturn.sqlite", "../wiki/measurements.md",
"&lt;image>"):
self.assertIn(expected, text)
def test_hosts_table_on_the_method_page(self) -> None:
page = sitemeasurements.with_hosts(f"x\n\n{sitemeasurements.HOSTS_MARKER}\n",
page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n",
{"items": [self.ITEM, {**self.ITEM, "host": "Test CPU, 12 CPUs, load 1 before, 2 after"}]})
self.assertIn("| Test CPU, 12 CPUs | 2 |", page)
self.assertNotIn(sitemeasurements.HOSTS_MARKER, page)
self.assertNotIn(sm.HOSTS_MARKER, page)
def test_the_explorer_reads_the_index(self) -> None:
source = (REPO_ROOT / "website" / "src" / "components" / "MeasurementsExplorer.astro").read_text()
@@ -404,37 +402,44 @@ class SitePages(unittest.TestCase):
read = set(re.findall(r"\b(?:item|i)\.([a-z_]+)", source)) - {"length"}
self.assertTrue(read)
self.assertLessEqual(read, set(self.ITEM), read - set(self.ITEM))
page = sitemeasurements.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40,
page = sm.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40,
"repo": REPO_URL})
self.assertIn("<MeasurementsExplorer />", page)
self.assertIn("downloads/measurements.sqlite", page)
self.assertIn(f"({REPO_URL}/tree/{'e' * 40}/measurements)", page)
def test_the_deploy_clones_the_measurements_at_the_pin(self) -> None:
workflow = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text()
clone = workflow.index("Clone the measurements at their pin")
step = workflow[clone:workflow.index("- name:", clone)]
self.assertIn("measurements-pin.json", step)
self.assertIn('fetch -q --depth 1 "$repo" "$commit"', step)
self.assertIn('checkout -q --detach "$commit"', step)
self.assertNotIn("uses:", step)
self.assertLess(clone, workflow.index("Validate data contracts"))
self.assertLess(clone, workflow.index("python scripts/generate_site.py --require-measurements"))
self.assertIn('- "measurements-pin.json"', workflow)
pin = sm.load_pin()
self.assertEqual(pin["repo"], REPO_URL)
def test_the_builds_read_the_records_in_this_repository(self) -> None:
for name in ("deploy-site.yml", "validate.yml"):
workflow = (REPO_ROOT / ".github" / "workflows" / name).read_text()
with self.subTest(workflow=name):
self.assertIn('- "measurements/**"', workflow)
self.assertNotIn("measurements-pin", workflow)
self.assertNotIn(".cache/measurements", workflow)
self.assertNotIn("--require-measurements", workflow)
self.assertFalse((REPO_ROOT / "measurements-pin.json").exists())
self.assertFalse((REPO_ROOT / "scripts" / "sync_measurements.py").exists())
def test_nothing_links_the_former_measurements_repository(self) -> None:
former = "retrobios-" + "measurements"
roots = ["wiki", "schemas", "scripts", "website/src", ".github", "tests", "measurements"]
suffixes = {".md", ".py", ".json", ".jsonl", ".yml", ".astro", ".ts", ".mjs"}
stray = [path.relative_to(REPO_ROOT).as_posix()
for root in roots for path in sorted((REPO_ROOT / root).rglob("*"))
if path.is_file() and path.suffix in suffixes
and former in path.read_text(encoding="utf-8", errors="replace")]
stray += [name for name in ("README.md", "CONTRIBUTING.md")
if former in (REPO_ROOT / name).read_text(encoding="utf-8")]
self.assertEqual(stray, [])
class Freshness(unittest.TestCase):
def test_pin_against_the_remote_head(self) -> None:
import check_freshness as cf # noqa: PLC0415
def test_no_measurements_area(self) -> None:
import check_freshness # noqa: PLC0415
import refresh_stale # noqa: PLC0415
pin = sm.load_pin()
self.assertEqual(cf.check_measurements(False, head=lambda _repo: pin["commit"]).status, cf.OK)
stale = cf.check_measurements(False, head=lambda _repo: "f" * 40)
self.assertEqual(stale.status, cf.STALE)
self.assertIn("sync_measurements.py --fetch --bump", stale.detail)
self.assertEqual(cf.check_measurements(True).status, cf.SKIPPED)
self.assertIn("measurements", cf.AREAS)
self.assertNotIn("measurements", check_freshness.AREAS)
self.assertNotIn("measurements", refresh_stale.AUTO_AREAS)
if __name__ == "__main__":
tests/test_site_r4_reference.py
+10 -19
View File
@@ -248,32 +248,23 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase):
def test_every_rendered_link_is_recorded(self):
record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*"))
import sync_measurements # noqa: PLC0415
import measurement_records # noqa: PLC0415
# A Tested block links the measurements repository at its pin. The
# record of source trees does not cover it: those paths are checked
# against the pinned tree of the clone itself.
measured_slug = urllib.parse.urlsplit(sync_measurements.load_pin()["repo"]).path.strip("/")
measured: dict[str, set[str]] = {}
# A Tested block links a record line and its data file in this
# repository. The records are append-only, so a file a permalink
# names is still in the tree.
records = set(measurement_records.tree_files(REPO_ROOT))
checked, stray = 0, []
for page in pages:
for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")):
if "/Abdess/retrobios/" in found[0]:
continue
key = f"{found['host']}/{found['slug']}@{found['pin']}"
path = urllib.parse.unquote(found["path"])
checked += 1
if found["slug"] == measured_slug:
if found["pin"] not in measured:
if not sync_measurements.has_commit(sync_measurements.CLONE, found["pin"]):
stray.append((page.name, key, "pin not in the measurements clone"))
measured[found["pin"]] = set()
continue
measured[found["pin"]] = set(
sync_measurements.tree_files(sync_measurements.CLONE, found["pin"]))
if path not in measured[found["pin"]]:
if "/Abdess/retrobios/" in found[0]:
if path.startswith(f"{measurement_records.RECORDS_DIR}/") and path not in records:
stray.append((page.name, key, path))
elif path not in record.get(key, ()):
continue
checked += 1
if path not in record.get(key, ()):
stray.append((page.name, key, path))
self.assertGreater(checked, 1000)
self.assertEqual(stray[:20], [])
website/src/components/MeasurementsExplorer.astro
+2 -3
View File
@@ -1,7 +1,7 @@
---
// The Measurements page: every test of the measurements index, filtered in
// the browser. Reads public/data/measurements/index.json, which
// scripts/sync_measurements.py writes from the records at the pinned commit.
// scripts/sitemeasurements.py writes from the records under measurements/.
// Each row keeps its method and the computer it ran on beside the result,
// and links its full record. The filters answer from the index alone: no
// SQL engine is loaded.
@@ -52,7 +52,6 @@ const base = import.meta.env.BASE_URL.replace(/\/?$/, '/');
host: string;
runs: number;
record_url: string;
data_url: string;
sqlite: string;
core: { library: string; commit: string | null };
};
@@ -124,7 +123,7 @@ const base = import.meta.env.BASE_URL.replace(/\/?$/, '/');
host.append(text('span', item.host), text('small', `Core ${item.core.library}${item.core.commit ? ` at ${item.core.commit.slice(0, 12)}` : ''}`));
tr.append(host);
const record = document.createElement('td');
record.append(link(item.record_url, 'Record'), document.createTextNode(' '), link(item.data_url, 'data'));
record.append(link(item.record_url, 'Record'));
record.append(text('small', item.runs === 1 ? '1 run' : `${item.runs} runs`));
tr.append(record);
return tr;
wiki/measurements.md
+3 -3
View File
@@ -38,9 +38,9 @@ shown without them.
## Where results are kept
Each result is a full record in the
[retrobios-measurements](https://github.com/Abdess/retrobios-measurements)
project. A record is never changed once added. A new run is added beside it,
and a correction names the record it replaces.
[`measurements/`](https://github.com/Abdess/retrobios/tree/main/measurements)
folder of this project. A record is never changed once added. A new run is
added beside it, and a correction names the record it replaces.
The results so far ran on these computers:
wiki/testing-guide.md
+1 -1
View File
@@ -57,7 +57,7 @@ Every module in `tests/`, by the area it guards.
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
| Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` |
| Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |
| Measurement records | a synthetic measurements repository built with git | `test_measurements` |
| Measurement records | synthetic record files, the committed `measurements/` against HEAD | `test_measurements` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
wiki/tools.md
+31 -49
View File
@@ -641,9 +641,8 @@ profile, `--record-coreinfo <core>` records its blob as reviewed. Data
directories are checked against their upstream, dump catalogs and romset
snapshots against the catalog they were read from (Redump DAT versions, the
No-Intro daily rebuild, the newest TOSEC pack, the newest MAME release, the
git blobs of the FBNeo DATs), the measurements pin against the head of the
measurements repository, and the CI toolchain against PyPI and the actions'
latest releases.
git blobs of the FBNeo DATs), and the CI toolchain against PyPI and the
actions' latest releases.
The `native` area needs no network. It reads the cache `export_native.py`
patches from and reports an original that is absent, that came from another
@@ -685,7 +684,6 @@ committed.
| `targets/<name>` | the target scraper |
| `data/<key>` | `refresh_data_dirs.py --key <key> --force` |
| `catalogs/mame recipes`, `catalogs/fbneo recipes` | `scraper/romset_dat_importer.py --source <source> --fetch` |
| `measurements/records pin` | `sync_measurements.py --fetch --bump` |
Rows that need a decision are listed and never run: a buildbot core with no
profile, a new `.info` file, the Redump, No-Intro and TOSEC packs, the CI
@@ -720,12 +718,12 @@ python scripts/core_probe.py run --core mednafen_saturn_libretro.so \
# timed repetitions after warm-up, written as a record
python scripts/core_probe.py bench --core core.so --content game.cue \
--warmup 300 --frames 3000 --repeat 5 --record .cache/measurements
--warmup 300 --frames 3000 --repeat 5 --record
# one system file swapped between runs, compared on one observable
python scripts/core_probe.py check --core core.so --content disc.cue \
--vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \
--observable video_sha1 --claim "..." --method "..." --record .cache/measurements
--observable video_sha1 --claim "..." --method "..." --record
```
Each repetition runs in a fresh process under `strace -f -e trace=openat,open`
@@ -747,7 +745,7 @@ strace the record says that tracing was unavailable.
| `--state-var SECTION.VAR` | a variable of a Mednafen save state, as an observable |
| `--state-dir` | keep the save state taken at the end of the first repetition |
| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record |
| `--record DIR` | append the record to the measurements clone `DIR`: one line of `records/<emulator>.jsonl` and its sidecar `data/<emulator>/<id>.json.gz`. A directory inside this repository's tracked tree is refused |
| `--record [DIR]` | append the record as one line of `measurements/<emulator>.jsonl` in the checkout `DIR`, by default this one. Commit it |
| `--emulator` | the profile the record is filed under. By default the profile whose `cores:` names the `.so`, else the core's library name |
| `--subject` | a slug naming the test, shared by its reruns. By default derived from the claim and method, or from the inputs of a run or bench |
| `--supersedes ID` | the id of an earlier record of the same emulator this one corrects |
@@ -768,53 +766,37 @@ host (CPU, kernel, Python, load before and after) and each input by hash.
Core log lines arrive through the libretro log interface. Floating-point
arguments of a log line cannot be read from Python and print as `<double>`.
A record is stored in two parts by `measurement_records.py`. The line keeps
the claim, method, result, timing statistics, host, core build, frontend
commit, inputs by hash and, per variant, the first repetition's observables
and files opened, under 20 KB. The sidecar keeps every repetition in full
(per-frame hashes, core logs, declared options), the raw timing values and
any compared value too large for a line, and its sha256 is in the line. The
id is the date and the first 12 hex digits of the sha256 of the line without
its id. Schemas: `core-probe-record.schema.json` for a line,
`core-probe-data.schema.json` for a sidecar. `join` rebuilds the full record
the run printed, `check_pair` checks a line against its sidecar.
A record is stored whole, as one line (`measurement_records.py`): the claim,
method, result, host, core build, frontend commit, inputs by hash, and every
repetition of every variant in full (per-frame hashes, core logs, declared
options), with the raw timing values the statistics come from. The id is the
date and the first 12 hex digits of the sha256 of the record without its id,
in canonical JSON. Schema: `core-probe-record.schema.json`.
### sync_measurements.py
### Measurement records
The records live in their own repository,
[Abdess/retrobios-measurements](https://github.com/Abdess/retrobios-measurements):
`records/<emulator>.jsonl`, one record per line, append-only, and
`data/<emulator>/<id>.json.gz`. This repository keeps only the pin,
`measurements-pin.json`, and reads the clone in `.cache/measurements/` at
that commit with git, whatever its working tree holds.
The records live in `measurements/` of this repository, one file per
emulator: `measurements/<emulator>.jsonl`, one full record per line in the
order added. A line is never rewritten or removed: a new run is a new line at
the end of its file, and a correction is a new line naming the record it
`supersedes`. Every line must hash to its id and validate against the record
schema, which `validate_schemas.py` checks. `tests/test_measurements.py`
fails when a line that HEAD holds has changed or gone. The seven records of
2026-10-27 were first written one JSON file per run, then split into a line
and a gzip data file, then merged back into one line. Each keeps its first
id, which hashes the first file (`migrated_from_version: 1`), so the line
still proves it rebuilds that file. Two branches that each add a line
merge cleanly: `.gitattributes` marks `measurements/*.jsonl` `merge=union`.
```bash
python scripts/sync_measurements.py --fetch # clone or update, check the remote
python scripts/sync_measurements.py --fetch --bump # then move the pin to its head
python scripts/sync_measurements.py --check-only # check every line and sidecar at the pin
python scripts/sync_measurements.py # write the site's measurement files
python scripts/sync_measurements.py --clone DIR --pin FILE # another clone or pin file
```
`--clone` and `--pin` name another clone and pin file than
`.cache/measurements/` and `measurements-pin.json`.
`--fetch` is the only step on the network. It refuses a remote whose history
changed or removed a line the pin holds, or rewrote a data file: a correction
is a new line naming the record it `supersedes`. Every line must hash to its
id and join its sidecar. Two branches that each add a line merge cleanly,
the measurements repository marking `records/*.jsonl` `merge=union`.
`generate_site.py` compiles the records at the pin into files it never
commits: `public/data/measurements/index.json` (one item per emulator and
subject, its latest record not superseded, with method, host and the
permalink of the line at the pin), `public/downloads/measurements.sqlite`
`generate_site.py` compiles the records into files it never commits
(`sitemeasurements.py`): `public/data/measurements/index.json` (one item per
emulator and subject, its latest record not superseded, with method, host and
the permalink of its line), `public/downloads/measurements.sqlite`
(normalised tables, indexed on emulator, system and claim, `meta` naming the
commit) and one `public/downloads/measurements/<emulator>.sqlite` per
emulator. Rows are sorted and nothing reads the clock, so two builds give
the same bytes. Without the pinned commit the site lists no measurements and
says so. `--require-measurements` makes that an error, and the deploy passes
it after a shallow fetch of the pinned commit.
emulator. A permalink names the last commit that touched `measurements/`, and
the build warns when the records on disk differ from that commit. Rows are
sorted and nothing reads the clock, so two builds give the same bytes.
### Other tools