Cross-check Arcanum support against dbmaker and the demo archive

tests/arcanum.sh drives Troika's vendored dbmaker.exe under wine in both
directions on a synthetic tree, then extracts the demo's real 13k-file
archive with both tools and diffs the trees. The demo download is
md5-pinned and the extracted archive is cached in CI.

Author: burner1024

.github/workflows/build.yml

@@ -52,7 +52,15 @@ jobs:
           DEBIAN_FRONTEND: noninteractive
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -y --no-install-recommends gcc-mingw-w64 musl-tools
+          sudo apt-get install -y --no-install-recommends gcc-mingw-w64 musl-tools p7zip-full
+
+      - name: Cache test fixtures
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            tests/rpu_v2.4.33.zip
+            tests/ArcanumDemo.dat
+          key: test-fixtures-rpu-v2.4.33-arcanum-demo-v1
 
       - name: Build binaries
         run: ./build.sh

.gitignore

@@ -6,12 +6,14 @@
 # Project-specific
 *.dat
 *.DAT
+/tmp/
 
 !/tests/
 /tests/*
 !/tests/*.sh
 !/tests/rpu.md5
 !/tests/dat2.exe
+!/tests/dbmaker.exe
 
 # IDE
 .vscode/

test.sh

@@ -9,6 +9,7 @@ cargo build --release --target x86_64-unknown-linux-musl
 cd tests
 ./non-ascii.sh
 ./rpu.sh
+./arcanum.sh
 
 # Put Fallout 1 critter.dat into tests/f1 to run this
 if [ -f f1/critter.dat ]; then

tests/arcanum.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -xeu -o pipefail
+
+# Work inside tests directory
+cd "$(dirname "$0")"
+
+# Load common variables and functions
+# shellcheck source=tests/common.sh
+source ./common.sh
+
+# Cross-check the Arcanum format against Troika's own dbmaker.exe
+# (usage: dbmaker [option] output input; -r recurse, -u unpack, -cN level)
+dbmaker() {
+	WINEDEBUG=-all wine dbmaker.exe "$@" 2>/dev/null
+}
+
+SRC_DIR="arcanum_src"
+DAT3_DAT="arcanum_dat3.dat"
+DB_DAT="arcanum_dbmaker.dat"
+DB_OUT="arcanum_db_out"
+DAT3_OUT="arcanum_dat3_out"
+
+# Build a small deterministic source tree: compressible files, a subdirectory,
+# and a file so small it stores uncompressed (zlib overhead exceeds its size)
+rm -rf "$SRC_DIR"
+mkdir -p "$SRC_DIR/data/sub"
+seq 1 2000 >"$SRC_DIR/data/numbers.txt"
+printf 'hi\n' >"$SRC_DIR/data/sub/tiny.txt"
+head -c 4096 /dev/zero >"$SRC_DIR/data/sub/zeros.bin"
+
+# dat3 writes, dbmaker reads
+rm -f "$DAT3_DAT"
+(cd "$SRC_DIR" && $DAT3 a --format arcanum -c 9 "../$DAT3_DAT" data)
+rm -rf "$DB_OUT"
+mkdir "$DB_OUT"
+(cd "$DB_OUT" && dbmaker -u "../$DAT3_DAT")
+diff -r "$SRC_DIR" "$DB_OUT"
+
+# dbmaker writes, dat3 auto-detects and reads
+rm -f "$DB_DAT"
+(cd "$SRC_DIR" && dbmaker -r -q "../$DB_DAT" data)
+$DAT3 l "$DB_DAT"
+rm -rf "$DAT3_OUT"
+$DAT3 x "$DB_DAT" -o "$DAT3_OUT"
+diff -r "$SRC_DIR" "$DAT3_OUT"
+
+# dat3 modifies the dbmaker-written archive; dbmaker reads the result
+echo "added later" >added.txt
+$DAT3 a "$DB_DAT" -t data added.txt
+$DAT3 d "$DB_DAT" "data/sub/zeros.bin"
+rm -rf "$DB_OUT"
+mkdir "$DB_OUT"
+(cd "$DB_OUT" && dbmaker -u "../$DB_DAT")
+diff "$DB_OUT/data/added.txt" added.txt
+if [ -e "$DB_OUT/data/sub/zeros.bin" ]; then
+	echo "Error: deleted file still present in archive"
+	exit 1
+fi
+
+# ── Real game data: the Arcanum demo's 30 MB archive (13k files) ──
+
+# Only the extracted archive is cached; the demo installer is fetched and
+# unpacked (RAR -> CAB -> DAT) solely to produce it
+DEMO_DAT="ArcanumDemo.dat"
+if [ ! -f "$DEMO_DAT" ]; then
+	DEMO_RAR="ArcanumDemo.rar"
+	if [ ! -f "$DEMO_RAR" ]; then
+		wget -nv -O "$DEMO_RAR" "https://archive.org/download/ArcanumDemo/ArcanumDemo.rar"
+	fi
+	echo "4a77a6cf6f801855cdfe0e7c61459a43  $DEMO_RAR" | md5sum -c
+	7z e -y "$DEMO_RAR" Setup1.cab
+	7z e -y Setup1.cab Arcanum.dat
+	mv Arcanum.dat "$DEMO_DAT"
+	rm -f Setup1.cab "$DEMO_RAR"
+fi
+echo "74e813d99a239c6ee04dd9fa807375a3  $DEMO_DAT" | md5sum -c
+
+# Both tools extract the real archive; the trees must be identical
+$DAT3 l "$DEMO_DAT"
+rm -rf demo_dat3 demo_db
+$DAT3 x "$DEMO_DAT" -o demo_dat3
+mkdir demo_db
+(cd demo_db && dbmaker -u "../$DEMO_DAT")
+diff -r demo_dat3 demo_db
+
+# dat3 modifies the real archive; dbmaker reads the result
+DEMO_MOD="demo_mod.dat"
+cp "$DEMO_DAT" "$DEMO_MOD"
+$DAT3 d "$DEMO_MOD" "WorldMap/WorldMap.mes"
+echo "demo test file" >demo_add.txt
+$DAT3 a "$DEMO_MOD" -t WorldMap demo_add.txt
+rm -rf demo_db
+mkdir demo_db
+(cd demo_db && dbmaker -u "../$DEMO_MOD")
+diff demo_db/WorldMap/demo_add.txt demo_add.txt
+if [ -e demo_db/WorldMap/WorldMap.mes ]; then
+	echo "Error: deleted file still present in archive"
+	exit 1
+fi
+
+# Clean up (keep the extracted DAT for the CI cache)
+rm -rf "$SRC_DIR" "$DB_OUT" "$DAT3_OUT" "$DAT3_DAT" "$DB_DAT" added.txt \
+	demo_dat3 demo_db "$DEMO_MOD" demo_add.txt