feat: build engine data at pinned commits

.gitignore

@@ -29,6 +29,14 @@ bios/Sony/PlayStation Vita/PSP2UPDAT.PUP
 # Large files stored as GitHub Release assets (additional)
 bios/Other/mame4droid-2024/data/history.db
 bios/Other/mame4droid-2024/data/.variants/history.db.615619e8
+bios/Interplay/Descent/d1x-rebirth/d1xr-opl3-music.dxa
+bios/Interplay/Descent/d1x-rebirth/d1xr-sc55-music.dxa
+bios/Game Engines/Sonic 3 AIR/sonic3-air/data/audioremaster.bin
+bios/Id Software/Return to Castle Wolfenstein/demomain/pak0.pk3
+bios/Id Software/Return to Castle Wolfenstein/demomain/.variants/pak0.pk3.82395c6e
+bios/Id Software/Doom 3/demo/demo00.pk4
+bios/Electronic Arts/Medal of Honor Allied Assault/mainta/pak3.pk3
+bios/Valve/Half-Life/valve/.variants/pak0.pak.2c57b84f
 bios/Arcade/MAME/artwork/snspell.zip
 bios/Arcade/MAME/MAME 0.174 Arcade XML.dat
 bios/Sony/PlayStation Vita/.variants/PSP2UPDAT.PUP.3ae832c9

emulators/devilutionx.yml

@@ -39,14 +39,22 @@ notes: |
 
   devilutionx.mpq is a build artefact packed from the repo's assets/
   directory with the smpq tool (CMake option BUILD_ASSETS_MPQ,
-  CMakeLists.txt:192-204). The compiled binary expects it next to the
+  CMakeLists.txt:192-204). CMake/Assets.cmake:46-238 lists the packed
-  executable on every platform except Android, Apple and 3DS/Switch
+  files (ASSETS_VERSION, arena, data, fonts, gendata, levels, lua,
-  (:509-516), where its contents are shipped loose. After loading the
+  plrgfx, txtdata and ui_art, plus data/inv/objcurs-widths.txt and the
-  core archives the engine reads the asset ASSETS_VERSION and aborts
+  ui_art PNGs outside SDL1 and Vita builds); each Translations/<lang>.po
-  with "Failed to load UI resources" unless its content is "1\n"
+  is compiled with msgfmt to <lang>.gmo and appended
-  (Source/init.cpp:54, 79-85, Source/diablo.cpp:1225-1239). The same
+  (CMake/Assets.cmake:8-29, 268-272). The archive is written with
-  test on fonts\VERSION applies to fonts.mpq when it is loaded
+  "smpq -A -M 1 -C BZIP2 -c" from the staged assets directory
-  (Source/init.cpp:98-114).
+  (CMake/Assets.cmake:287-294). mods/hf.mpq is built the same way from
+  the mods/hf file list (CMake/Mods.cmake:8-33, 58-67). The compiled
+  binary expects devilutionx.mpq next to the executable on every
+  platform except Android, Apple and 3DS/Switch (:509-516), where its
+  contents are shipped loose. After loading the core archives the engine
+  reads the asset ASSETS_VERSION and aborts with "Failed to load UI
+  resources" unless its content is "1\n" (Source/init.cpp:54, 79-85,
+  Source/diablo.cpp:1225-1239). The same test on fonts\VERSION applies
+  to fonts.mpq when it is loaded (Source/init.cpp:98-114).
 
   fonts.mpq is the CJK + language assets add-on from the
   devilutionx-assets release; English installs do not need it.
@@ -124,9 +132,9 @@ files:
     system: diablo
     category: game_data
     required: true
-    description: "Engine assets (UI art, fonts, data tables, Lua scripts) packed from the repository's assets/ directory"
+    description: "Engine assets (UI art, fonts, data tables, Lua scripts, compiled translations) packed from the repository's assets/ directory"
-    note: "Loaded first on every platform except Android, Apple, 3DS and Switch. Its files are also found in an assets/ directory next to the binary when the build installs them loose. The engine aborts when the asset ASSETS_VERSION is absent or is not \"1\\n\"."
+    note: "Loaded first on every platform except Android, Apple, 3DS and Switch. Its files are also found in an assets/ directory next to the binary when the build installs them loose. The engine aborts when the asset ASSETS_VERSION is absent or is not \"1\\n\". Built by CMake with smpq (MPQ version 1, BZIP2, no attributes) from the file list of CMake/Assets.cmake plus one <lang>.gmo per translation."
-    source_ref: "Source/engine/assets.cpp:509-516, 223-226, Source/init.cpp:79-85, Source/diablo.cpp:1225-1239"
+    source_ref: "Source/engine/assets.cpp:509-516, 223-226, Source/init.cpp:54, 79-85, Source/diablo.cpp:1225-1239, CMake/Assets.cmake:46-224, 287-294"
 
   - name: "fonts.mpq"
     path: "devilutionx/fonts.mpq"
@@ -169,5 +177,5 @@ files:
     category: game_data
     required: false
     description: "Hellfire mod archive packed from the repository's mods/hf directory"
-    note: "mods/<name>.mpq is opened for each active mod; hf is activated when hellfire.mpq is present. A mod with no archive and no loose directory is served from the core archives."
+    note: "mods/<name>.mpq is opened for each active mod; hf is activated when hellfire.mpq is present. A mod with no archive and no loose directory is served from the core archives. Built by CMake with smpq (MPQ version 1, BZIP2, no attributes) from the hellfire_mod file list."
-    source_ref: "Source/engine/assets.cpp:748-764, Source/options.cpp:77-79, CMakeLists.txt:204"
+    source_ref: "Source/engine/assets.cpp:748-764, Source/options.cpp:77-79, CMakeLists.txt:204, CMake/Mods.cmake:8-33, 58-66"

emulators/gzdoom.yml

@@ -343,7 +343,8 @@ files:
     category: game_data
     system: doom
     required: false
-    source_ref: "wadsrc_extra/static/iwadinfo.txt:71, 699"
+    source_ref: "wadsrc_extra/static/iwadinfo.txt:65-75, 699, src/d_iwad.cpp:351-394, src/common/filesystem/source/resourcefile.cpp:156-174"
+    note: "Identified by the MAP01 map and the HACX-E lump, whatever the container: every candidate is opened by its magic, so a zip under this name is read as a pk3. The Hacx 2.0 repository builds that zip from src/, where hacx-e.txt is the HACX-E lump and maps/MAP01.wad the first map."
 
   - name: "action2.wad"
     category: game_data

emulators/sonic3-air.yml

@@ -38,15 +38,33 @@ notes: |
        Oxygen/sonic3air/source/sonic3air/ConfigurationImpl.cpp:33-47,
        Oxygen/sonic3air/oxygenproject.json:9-20
 
-  The engine ships four data packages built from Oxygen/sonic3air/data/
+  The engine ships four data packages (enginedata.bin, gamedata.bin,
-  (enginedata.bin, gamedata.bin, audiodata.bin, audioremaster.bin). Three
+  audiodata.bin, audioremaster.bin). Three are required and part of the
-  are required and part of the release; audioremaster.bin is optional and
+  release; audioremaster.bin is optional and downloaded on demand from
-  downloaded on demand from https://sonic3air.org/download/audioremaster.bin
+  https://sonic3air.org/download/audioremaster.bin when the user enables
-  when the user enables the remastered soundtrack.
+  the remastered soundtrack.
   ref: Oxygen/sonic3air/oxygenproject.json:32-38,
        Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:115-118,
        Oxygen/sonic3air/source/sonic3air/audio/RemasteredMusicDownload.cpp:48-62
 
+  The packages are written by the game binary itself when run with -pack
+  from Oxygen/sonic3air/ (PackageBuilder::performPacking), which the
+  mastering scripts call before copying the results to the release image.
+    gamedata.bin       data/ without data/audio/ and data/metadata.json
+    audiodata.bin      data/audio/original/
+    audioremaster.bin  data/audio/remastered/
+    enginedata.bin     ../oxygenengine/data/
+  Each package is a flat container: "OPCK", format version 3, content
+  version BUILD_NUMBER, then one entry per file keyed by its path relative
+  to the base directory, in std::map order. Packing reads the data
+  directories only; the ROM is not involved.
+  ref: Oxygen/sonic3air/source/sonic3air/main.cpp:92-97,
+       Oxygen/sonic3air/source/sonic3air/helper/PackageBuilder.cpp:16-63,
+       Oxygen/oxygenengine/source/oxygen/file/FilePackage.cpp:44-151,
+       Oxygen/oxygenengine/source/oxygen/file/FilePackage.h:28-30,
+       Oxygen/sonic3air/source/sonic3air/version.inc:1,
+       Oxygen/sonic3air/___internal/mastering/mastering_linux.sh:48-52
+
   Each package is opened from the game data directory, data/ under the
   install directory, then from data/ under the app-data directory, where
   the downloader saves audioremaster.bin. Package paths below are written
@@ -81,24 +99,25 @@ files:
     category: game_data
     system: sonic3-air
     required: true
-    bundled: true
+    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:115, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651, Oxygen/sonic3air/source/sonic3air/helper/PackageBuilder.cpp:57-62"
-    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:115, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651"
     note: |
-      Oxygen Engine data package. A required package that cannot be opened
+      Oxygen Engine data package, packed from Oxygen/oxygenengine/data/
-      closes the application, unless the data is present unpacked
+      (fonts, images, shaders, sprites, oxygen_icon.png). A required
-      (data/images/icon.png exists). The engine then checks for
+      package that cannot be opened closes the application, unless the
-      oxygen_icon.png in the engine data and stops when it is absent
+      data is present unpacked (data/images/icon.png exists). The engine
-      (EngineMain.cpp:584-593).
+      then checks for oxygen_icon.png in the engine data and stops when it
+      is absent (EngineMain.cpp:584-593).
 
   - name: "gamedata.bin"
     path: "sonic3-air/data/gamedata.bin"
     category: game_data
     system: sonic3-air
     required: true
-    bundled: true
+    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:116, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651, Oxygen/sonic3air/source/sonic3air/helper/PackageBuilder.cpp:36-41"
-    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:116, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651"
     note: |
-      Game data package. A required package that cannot be opened closes
+      Game data package, packed from Oxygen/sonic3air/data/ without the
+      audio directory and metadata.json (binary tables, fonts, images,
+      rawdata, sprites). A required package that cannot be opened closes
       the application, unless the data is present unpacked
       (data/images/icon.png exists).
 
@@ -107,12 +126,12 @@ files:
     category: game_data
     system: sonic3-air
     required: true
-    bundled: true
+    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:117, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651, Oxygen/sonic3air/source/sonic3air/helper/PackageBuilder.cpp:43-48"
-    source_ref: "Oxygen/sonic3air/source/sonic3air/EngineDelegate.cpp:117, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:603-617, Oxygen/oxygenengine/source/oxygen/engine/EngineMain.cpp:643-651"
     note: |
-      Audio data package. A required package that cannot be opened closes
+      Audio data package, packed from Oxygen/sonic3air/data/audio/original/
-      the application, unless the data is present unpacked
+      (emulated soundtrack definitions and prototype SMPS tracks). A
-      (data/images/icon.png exists).
+      required package that cannot be opened closes the application,
+      unless the data is present unpacked (data/images/icon.png exists).
 
   - name: "audioremaster.bin"
     path: "sonic3-air/data/audioremaster.bin"

emulators/tr1x.yml

@@ -43,6 +43,7 @@ files:
   # Enumerated by data/trx/ship/games/tr1/gameflow.json5 "levels" array.
   - name: "title.phd"
     path: "games/tr1/levels/title.phd"
+    unsourceable: "retail data the collection does not hold; a same-named file of another game must not stand in"
     required: true
     category: game_data
     source_ref: "data/trx/ship/games/tr1/gameflow.json5:38"
@@ -62,6 +63,7 @@ files:
 
   - name: "level2.phd"
     path: "games/tr1/levels/level2.phd"
+    unsourceable: "retail data the collection does not hold; a same-named file of another game must not stand in"
     required: true
     category: game_data
     source_ref: "data/trx/ship/games/tr1/gameflow.json5:116"
@@ -631,46 +633,52 @@ files:
   # Known game tr1-ub, extends tr1 (src/trx/game/shell/mod.c:30-31).
   # Enumerated by data/trx/ship/games/tr1-ub/gameflow.json5 "levels" array.
   # Its title, FMVs and music fall back to games/tr1.
+  # The project's installer downloads the levels from its own host as
+  # tr1-ub-vanilla.zip (original edition) or tr1-ub-music.zip (fan-patched
+  # edition) and copies data/*.phd to games/tr1-ub/levels/; tools/download_assets
+  # fetches tr1-ub.zip, which carries the expansion's images only.
   - name: "egypt.phd"
     path: "games/tr1-ub/levels/egypt.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:63"
+    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:63, tools/installer/TRX_Installer/InstallComponentFactory.cs:72-81, tools/installer/TRX_Installer/InstallMappings.cs:18-22"
-    note: "Return to Egypt"
+    note: "Return to Egypt. The installer offers the original edition and a fan-patched edition of the level set."
 
   - name: "cat.phd"
     path: "games/tr1-ub/levels/cat.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:87"
+    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:87, tools/installer/TRX_Installer/InstallComponentFactory.cs:72-81, tools/installer/TRX_Installer/InstallMappings.cs:18-22"
-    note: "Temple of the Cat"
+    note: "Temple of the Cat. The installer offers the original edition and a fan-patched edition of the level set."
 
   - name: "end.phd"
     path: "games/tr1-ub/levels/end.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:110"
+    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:110, tools/installer/TRX_Installer/InstallComponentFactory.cs:72-81, tools/installer/TRX_Installer/InstallMappings.cs:18-22"
-    note: "Atlantean Stronghold"
+    note: "Atlantean Stronghold. The installer offers the original edition and a fan-patched edition of the level set."
 
   - name: "end2.phd"
     path: "games/tr1-ub/levels/end2.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:132"
+    source_ref: "data/trx/ship/games/tr1-ub/gameflow.json5:132, tools/installer/TRX_Installer/InstallComponentFactory.cs:72-81, tools/installer/TRX_Installer/InstallMappings.cs:18-22"
-    note: "The Hive"
+    note: "The Hive. The installer offers the original edition and a fan-patched edition of the level set."
 
   # PC demo
   # Known game tr1-demo-pc, extends tr1 (src/trx/game/shell/mod.c:32-33).
+  # tools/download_assets and the installer fetch tr1-demo-pc.zip from the
+  # project's host; it holds games/tr1-demo-pc/levels/{title,level2}.phd.
   - name: "title.phd"
     path: "games/tr1-demo-pc/levels/title.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-demo-pc/gameflow.json5:38, src/trx/game/game_flow/reader.c:903-909, src/trx/game/paths.c:123-132"
+    source_ref: "data/trx/ship/games/tr1-demo-pc/gameflow.json5:38, src/trx/game/game_flow/reader.c:903-909, src/trx/game/paths.c:123-132, tools/download_assets:67, tools/installer/TRX_Installer/InstallComponentFactory.cs:83-87"
     note: "Title level of the PC demo. A title level is also looked up in the levels directory of the game it extends, so games/tr1/levels/title.phd is used when this one is absent."
 
   - name: "level2.phd"
     path: "games/tr1-demo-pc/levels/level2.phd"
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr1-demo-pc/gameflow.json5:56, src/trx/game/level/format/format_tr1.c:74-99, 186-191"
+    source_ref: "data/trx/ship/games/tr1-demo-pc/gameflow.json5:56, src/trx/game/level/format/format_tr1.c:74-99, 186-191, tools/download_assets:67, tools/installer/TRX_Installer/InstallComponentFactory.cs:83-87"
     note: "City of Vilcabamba in the PC demo layout, which stores the palette before the cameras. The demo game does not load without it."

emulators/tr2x.yml

@@ -46,6 +46,7 @@ files:
   # Soundbank
   - name: "main.sfx"
     path: "games/tr2/main.sfx"
+    unsourceable: "retail data the collection does not hold; a same-named file of another game must not stand in"
     required: true
     category: game_data
     source_ref: "data/trx/ship/games/tr2/gameflow.json5:33"
@@ -55,6 +56,7 @@ files:
   # Enumerated by data/trx/ship/games/tr2/gameflow.json5 "levels" array.
   - name: "title.tr2"
     path: "games/tr2/levels/title.tr2"
+    unsourceable: "retail data the collection does not hold; a same-named file of another game must not stand in"
     required: true
     category: game_data
     source_ref: "data/trx/ship/games/tr2/gameflow.json5:15"
@@ -649,12 +651,16 @@ files:
   # Known game tr2-gm, extends tr2 (src/trx/game/shell/mod.c:37-38).
   # Enumerated by data/trx/ship/games/tr2-gm/gameflow.json5 "levels" array.
   # Its FMVs and music fall back to games/tr2.
+  # tools/download_assets and the installer fetch tr2-gm.zip from the
+  # project's host; it holds games/tr2-gm/main.sfx and games/tr2-gm/levels/*.tr2.
+  # Installing from a retail directory maps data/main_gm.sfx to
+  # games/tr2-gm/main_gm.sfx and data/*.tr2 to games/tr2-gm/levels/.
   - name: "main.sfx"
     path: "games/tr2-gm/main.sfx"
     aliases: ["main_gm.sfx"]
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr2-gm/gameflow.json5:34, src/trx/game/game_flow/reader.c:198-219, src/trx/game/paths.c:203-212"
+    source_ref: "data/trx/ship/games/tr2-gm/gameflow.json5:34, src/trx/game/game_flow/reader.c:198-219, src/trx/game/paths.c:203-212, tools/download_assets:71, tools/installer/TRX_Installer/InstallMappings.cs:35-40"
     note: "Sample bank of the expansion. main_gm.sfx is tried first, each name in the expansion directory then in games/tr2."
 
   - name: "title.tr2"
@@ -662,7 +668,7 @@ files:
     aliases: ["title_gm.tr2"]
     required: false
     category: game_data
-    source_ref: "data/trx/ship/games/tr2-gm/gameflow.json5:16, src/trx/game/game_flow/reader.c:198-219, 903-909, src/trx/game/paths.c:123-132"
+    source_ref: "data/trx/ship/games/tr2-gm/gameflow.json5:16, src/trx/game/game_flow/reader.c:198-219, 903-909, src/trx/game/paths.c:123-132, tools/download_assets:71, tools/installer/TRX_Installer/InstallMappings.cs:35-40"
     note: "Title level of the expansion. title_gm.tr2 is tried first, each name in the expansion levels directory then in games/tr2/levels."
 
   - name: "level1.tr2"