diff --git a/.github/workflows/synctest.yml b/.github/workflows/synctest.yml new file mode 100644 index 0000000000..89fa59a471 --- /dev/null +++ b/.github/workflows/synctest.yml @@ -0,0 +1,273 @@ +# Runs the Beyond All Reason "synctest" (/luarules synctest) against an engine +# commit on all supported platforms (amd64-linux, arm64-linux, amd64-windows). +# +# This workflow looks up the most recent successful "Build Engine v2" run for +# the target commit and downloads platform-specific artifacts. If no such run +# exists, the workflow fails fast. +# TODO: trigger after every engine build +# +# The cross-platform-check job compares hashes across architectures and is +# intended as a merge-blocking required check. +# +# See test/synctest/README.md for background and how to run locally. +name: Sync test + +on: + workflow_dispatch: + inputs: + commit: + description: 'Engine commit SHA to test. Leave blank to test the head of the ref selected in the picker above.' + type: string + default: '' + +permissions: + # actions:read is required to list workflow runs and download artifacts + # produced by a different workflow run (the Build Engine v2 run for the + # target commit). + actions: read + contents: read + +defaults: + run: + shell: bash -euo pipefail {0} + +env: + MAP: "Jade Empress 1.41" + GAME: "Beyond All Reason test-30212-63c6ded" + HASH_FILE: "synctest_synchash.json" + +jobs: + setup: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + sha: ${{ steps.resolve.outputs.sha }} + run_id: ${{ steps.engine-run.outputs.run_id }} + platforms: ${{ steps.platforms.outputs.json }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Define target platforms + id: platforms + # Single source of truth for the platform matrix: needed so we can check + # if we have all platforms even if a later setup step fails. + run: | + # jq -c -n validates the JSON and emits it compact, as GITHUB_OUTPUT requires. + json=$(jq -c -n '[ + {"platform": "amd64-linux", "runs-on": "ubuntu-latest", "bin-suffix": ""}, + {"platform": "arm64-linux", "runs-on": "ubuntu-24.04-arm", "bin-suffix": ""}, + {"platform": "amd64-windows", "runs-on": "windows-latest", "bin-suffix": ".exe"} + ]') + echo "json=$json" >> "$GITHUB_OUTPUT" + + - name: Resolve target commit + id: resolve + env: + INPUT_COMMIT: ${{ inputs.commit }} + REPO: ${{ github.repository }} + DEFAULT_SHA: ${{ github.sha }} + run: | + target="${INPUT_COMMIT:-$DEFAULT_SHA}" + + # Normalize whatever the user pasted (full sha, short sha, branch, tag) + # into a full 40-char SHA so the run lookup below is unambiguous. + full_sha=$(gh api "repos/$REPO/commits/$target" --jq '.sha') + echo "Resolved target commit: $full_sha" + echo "sha=$full_sha" >> "$GITHUB_OUTPUT" + + - name: Find Build Engine v2 run for this commit + id: engine-run + env: + REPO: ${{ github.repository }} + TARGET_SHA: ${{ steps.resolve.outputs.sha }} + run: | + run_id=$(gh api \ + "repos/$REPO/actions/workflows/engine-build.yml/runs?head_sha=$TARGET_SHA&status=success" \ + --jq '.workflow_runs[0].id') + if [ -z "$run_id" ] || [ "$run_id" = "null" ]; then + echo "::error::No successful 'Build Engine v2' run found for commit $TARGET_SHA." + echo "::error::Trigger Build Engine v2 on that commit first (push it, or dispatch engine-build.yml manually) and retry." + exit 1 + fi + echo "Using engine-build run $run_id" + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + + synctest: + needs: setup + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.setup.outputs.platforms) }} + steps: + - name: Checkout workflow definition and test inputs + # Uses the ref picked in the dispatch UI (github.sha), NOT the target + # commit — so the startscript and any other test assets come from the + # workflow-definition ref. The target commit only controls which engine + # artifact gets downloaded below. + uses: actions/checkout@v4 + + - name: Download engine artifact (${{ matrix.platform }}) + uses: actions/download-artifact@v4 + with: + pattern: engine-artifacts-${{ matrix.platform }}* + run-id: ${{ needs.setup.outputs.run_id }} + github-token: ${{ github.token }} + path: engine-artifact + merge-multiple: true + + - name: Extract install tree + run: | + mkdir -p install + + # Engine is packaged with INSTALL_PORTABLE=ON (CMake default, see + # CMakeLists.txt:22), so binaries live at the install-tree root: + # install/spring-headless, install/pr-downloader, install/spring, ... + # The archive name comes from docker-build-v2/scripts/package.sh; that + # script also emits a -dbgsym.tar.zst, which this glob excludes. + # + # Windows has 7z.exe, and ubuntu currently has 7z/7zz/7za but may later + # drop to just 7zz/7za. + sevenzip=$(command -v 7z || command -v 7zz || command -v 7za || true) + if [ -z "$sevenzip" ]; then + echo "::error::no 7-Zip binary (7z/7zz/7za) found on runner"; exit 1 + fi + "$sevenzip" x engine-artifact/recoil_*_${{ matrix.platform }}*.7z -oinstall + + - name: Restore BAR content cache + uses: actions/cache@v4 + with: + # Content-addressed and immutable, so all three platform legs share one + # entry. packages/ (the .sdp manifests) is required alongside pool/ + # for the engine to mount the game. Deliberately NOT the whole write-dir: + # everything else the engine writes here is build- and platform-specific. + # rapid/ is excluded too — mutable repo metadata, a few MB, refetched so + # the pin is genuinely re-resolved each run. + path: | + bar-data/pool + bar-data/packages + bar-data/maps + # IMPORTANT: + # Bump this key in lockstep with any change to the pinned + # gametype/mapname. See test/synctest/README.md. + key: bar-data-${{ env.GAME }}-${{ env.MAP }} + + - name: Fetch pinned BAR assets (no-op on warm cache) + run: | + ./install/pr-downloader${{ matrix.bin-suffix }} \ + --filesystem-writepath "$PWD/bar-data" \ + --download-game "$GAME" \ + --download-map "$MAP" + env: + PRD_RAPID_USE_STREAMER: "false" + PRD_RAPID_REPO_MASTER: "https://repos-cdn.beyondallreason.dev/repos.gz" + PRD_HTTP_SEARCH_URL: "https://files-cdn.beyondallreason.dev/find" + + - name: Render startscript + run: | + # The checked-in startscript is a template: @VERSION@ is the version + # suffix of the pinned GAME (text after the last space, e.g. + # "test-29932-21b3bb0") and @MAPNAME@ is MAP. + sed -e "s/@VERSION@/${GAME##* }/g" -e "s/@MAPNAME@/$MAP/g" \ + test/synctest/synctest-startscript.txt > startscript.txt + + # Fail loudly here rather than letting the engine fail later with an + # opaque "can't find game/map" error. + if grep -qE '@VERSION@|@MAPNAME@' startscript.txt; then + echo "::error::startscript template was not fully rendered — leftover placeholders:" + grep -nE '@VERSION@|@MAPNAME@' startscript.txt + exit 1 + fi + grep -nE 'gametype=|mapname=' startscript.txt + + - name: Run synctest + run: | + ./install/spring-headless${{ matrix.bin-suffix }} --isolation \ + --write-dir "$PWD/bar-data" startscript.txt 2>&1 | tee infolog.txt + + - name: Assert sync-hash file produced + run: | + digest=$(jq -r '.digest // empty' "bar-data/$HASH_FILE" 2>/dev/null || true) + if [ -z "$digest" ]; then + echo "::error::bar-data/$HASH_FILE is missing, empty, or has no .digest — the synctest produced no checksum output" + exit 1 + fi + echo "Sync hash digest: $digest" + + - name: Upload run artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: synctest-${{ matrix.platform }}-${{ needs.setup.outputs.sha }} + path: | + infolog.txt + bar-data/*.log + bar-data/${{ env.HASH_FILE }} + retention-days: 30 + if-no-files-found: ignore + + # Collects sync hashes from all synctest runs across platforms and fails if + # hashes diverge between architectures (cross-platform desync detection). + # This job is intended to be a required/merge-blocking status check. + cross-platform-check: + if: ${{ !cancelled() }} + needs: [setup, synctest] + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + TESTED_SHA: ${{ needs.setup.outputs.sha }} + steps: + - name: Assert setup produced a platform list + env: + PLATFORMS_JSON: ${{ needs.setup.outputs.platforms }} + run: | + if [ -z "$PLATFORMS_JSON" ]; then + echo "::error::setup produced no platform list — it likely failed before it ran; check the setup job." + exit 1 + fi + + - name: Download current run artifacts + # If every platform leg died before uploading, the pattern matches + # nothing and this step errors. That is not the failure we want to + # report, so swallow it and let the MISSING check below speak instead. + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: synctest-*-${{ env.TESTED_SHA }} + path: artifacts + + - name: Compare sync hashes across platforms + env: + PLATFORMS_JSON: ${{ needs.setup.outputs.platforms }} + run: | + # Check from the EXPECTED platform list, so if artifact uploading fails + # we will be able to detect that its missing. + for platform in $(jq -r '.[].platform' <<<"$PLATFORMS_JSON"); do + digest=$(jq -r '.digest // empty' \ + "artifacts/synctest-$platform-$TESTED_SHA/bar-data/$HASH_FILE" 2>/dev/null || true) + echo "$platform ${digest:-MISSING}" + done > synchashes.txt + + echo "Cross-platform determinism check:" + cat synchashes.txt + + if grep -qw MISSING synchashes.txt; then + echo "::error::sync hash MISSING on at least one platform — see the per-platform jobs" + exit 1 + fi + digests=$(awk '{print $2}' synchashes.txt | sort -u) + if [ "$(wc -l <<<"$digests")" -gt 1 ]; then + echo "::error::CROSS-PLATFORM DESYNC — hashes differ across architectures; the engine is not deterministic across platforms" + exit 1 + fi + echo "OK across all platforms ($digests)" + + # A platform can write the hash and then fail, but + # `!cancelled()` means we still run the comparison. + # So let's do a final check to make sure they all passed. + - name: Assert all synctest jobs succeeded + if: ${{ needs.synctest.result != 'success' }} + run: | + echo "::error::synctest matrix result is '${{ needs.synctest.result }}' — a platform leg did not succeed; see the per-platform jobs" + exit 1 diff --git a/rts/Lua/LuaConstEngine.cpp b/rts/Lua/LuaConstEngine.cpp index 0d38c1ffbf..579a07102f 100644 --- a/rts/Lua/LuaConstEngine.cpp +++ b/rts/Lua/LuaConstEngine.cpp @@ -45,7 +45,6 @@ * @field wordSize number Indicates the build type always 64 these days * @field gameSpeed number Number of simulation gameframes per second * @field textColorCodes TextColorCode Table containing keys that represent the color code operations during font rendering - * @field isHeadless boolean? Whether this is a headless engine build. Not available in synced */ bool LuaConstEngine::PushEntries(lua_State* L) diff --git a/rts/Lua/LuaConstPlatform.cpp b/rts/Lua/LuaConstPlatform.cpp index 7262bd7d5a..9c23cd29db 100644 --- a/rts/Lua/LuaConstPlatform.cpp +++ b/rts/Lua/LuaConstPlatform.cpp @@ -151,5 +151,12 @@ bool LuaConstPlatform::PushEntries(lua_State* L) /*** @field Platform.isHeadless boolean Is this a headless build which only simulates and doesnt offer interactive IO? */ LuaPushNamedBool(L, "isHeadless", SpringVersion::IsHeadless()); + /*** @field Platform.hasSyncChecksums boolean Whether the engine was built with sync-check support (i.e. Spring.GetPrevFrameSyncChecksum() returns a meaningful value). */ + #ifdef SYNCCHECK + LuaPushNamedBool(L, "hasSyncChecksums", true); + #else + LuaPushNamedBool(L, "hasSyncChecksums", false); + #endif + return true; } diff --git a/rts/Lua/LuaUnsyncedRead.cpp b/rts/Lua/LuaUnsyncedRead.cpp index f1615b8079..8b4e95a9e0 100644 --- a/rts/Lua/LuaUnsyncedRead.cpp +++ b/rts/Lua/LuaUnsyncedRead.cpp @@ -75,6 +75,7 @@ #include "System/Sound/ISound.h" #include "System/Sound/ISoundChannels.h" #include "System/StringUtil.h" +#include "System/Sync/SyncChecker.h" #include "System/Misc/SpringTime.h" #include "System/ScopedResource.h" #include "System/Math/NURBS.h" @@ -120,6 +121,7 @@ bool LuaUnsyncedRead::PushEntries(lua_State* L) REGISTER_LUA_CFUNC(GetGameSecondsInterpolated); REGISTER_LUA_CFUNC(GetLastUpdateSeconds); REGISTER_LUA_CFUNC(GetVideoCapturingMode); + REGISTER_LUA_CFUNC(GetPrevFrameSyncChecksum); REGISTER_LUA_CFUNC(GetNumDisplays); REGISTER_LUA_CFUNC(GetViewGeometry); @@ -1236,6 +1238,35 @@ int LuaUnsyncedRead::GetVideoCapturingMode(lua_State* L) } +/*** + * + * Returns the engine's sync checksum for the previous simframe, + * useful for testing. The returned string is NOT convertible to + * a number within Lua. + * + * Returns a dummy value if `Platform.hasSyncChecksums` is false, + * or if no frames were processed yet. + * + * @function Spring.GetPrevFrameSyncChecksum + * + * @return string checksum + */ +int LuaUnsyncedRead::GetPrevFrameSyncChecksum(lua_State* L) +{ +#ifdef SYNCCHECK + unsigned checksum = CSyncChecker::GetPrevChecksum(); +#else + unsigned checksum = 0; +#endif + + char buf[9]; + snprintf(buf, sizeof(buf), "%08x", checksum); + lua_pushstring(L, buf); + + return 1; +} + + /****************************************************************************** * Unit attributes * @section unitattributes diff --git a/rts/Lua/LuaUnsyncedRead.h b/rts/Lua/LuaUnsyncedRead.h index 2d26b5544f..3926912fdc 100644 --- a/rts/Lua/LuaUnsyncedRead.h +++ b/rts/Lua/LuaUnsyncedRead.h @@ -32,6 +32,7 @@ class LuaUnsyncedRead { static int GetGameSecondsInterpolated(lua_State* L); static int GetLastUpdateSeconds(lua_State* L); static int GetVideoCapturingMode(lua_State* L); + static int GetPrevFrameSyncChecksum(lua_State* L); static int GetNumDisplays(lua_State* L); static int GetViewGeometry(lua_State* L); diff --git a/rts/Net/NetCommands.cpp b/rts/Net/NetCommands.cpp index 124d44b817..55d5d0071d 100644 --- a/rts/Net/NetCommands.cpp +++ b/rts/Net/NetCommands.cpp @@ -623,6 +623,12 @@ void CGame::ClientReadNet() ASSERT_SYNCED(CSyncChecker::GetChecksum()); clientNet->Send(CBaseNetProtocol::Get().SendSyncResponse(gu->myPlayerNum, gs->frameNum, CSyncChecker::GetChecksum())); + // Cache the just-closed frame's checksum so Lua can read a + // stable value during the next sim frame via + // Spring.GetPrevFrameSyncChecksum(). Must run before the + // 4096-frame reset below so we capture the pre-reset value. + CSyncChecker::SetPrevChecksum(CSyncChecker::GetChecksum()); + // buffer all checksums, so we can check sync later between demo & local if (haveServerDemo) localSyncChecksums[gs->frameNum] = CSyncChecker::GetChecksum(); diff --git a/rts/System/Sync/SyncChecker.cpp b/rts/System/Sync/SyncChecker.cpp index 4063faff10..edb0cb02e4 100644 --- a/rts/System/Sync/SyncChecker.cpp +++ b/rts/System/Sync/SyncChecker.cpp @@ -11,6 +11,7 @@ unsigned CSyncChecker::g_checksum; +unsigned CSyncChecker::g_prevChecksum; int CSyncChecker::inSyncedCode; void CSyncChecker::NewFrame() diff --git a/rts/System/Sync/SyncChecker.h b/rts/System/Sync/SyncChecker.h index 35aa1be6d7..3fb996c262 100644 --- a/rts/System/Sync/SyncChecker.h +++ b/rts/System/Sync/SyncChecker.h @@ -31,6 +31,8 @@ class CSyncChecker { * Keeps a running checksum over all assignments to synced variables. */ static unsigned GetChecksum() { return g_checksum; } + static unsigned GetPrevChecksum() { return g_prevChecksum; } + static void SetPrevChecksum(unsigned v) { g_prevChecksum = v; } static void NewFrame(); static void debugSyncCheckThreading(); static void Sync(uint32_t val); @@ -48,6 +50,11 @@ class CSyncChecker { */ static unsigned g_checksum; + /** + * Final checksum of the previous simulation frame. + */ + static unsigned g_prevChecksum; + /** * @brief in synced code * diff --git a/test/synctest/README.md b/test/synctest/README.md new file mode 100644 index 0000000000..30af8a81fe --- /dev/null +++ b/test/synctest/README.md @@ -0,0 +1,42 @@ +# Synctest + +Engine CI sync test driven by BAR's synctest gadget (see beyond-all-reason/Beyond-All-Reason#7440 and #7445). The `.github/workflows/synctest.yml` workflow runs a single seeded scenario on each supported platform (amd64-linux, arm64-linux, amd64-windows) on top of a pinned BAR game + map and asserts that a deterministic sync-hash JSON file lands in the engine's write-dir. The `cross-platform-check` job then verifies all platforms produced identical digests, catching architecture-specific desyncs. Background: beyond-all-reason/RecoilEngine#2910, beyond-all-reason/RecoilEngine#2906. + +The gadget itself rotates through unit categories (bots, tanks, fighters, bombers, hover, subs, ships, spiders) internally, so one scenario is enough — no per-category scripts needed. + +## Running locally + +Build the engine, make sure `spring-headless` and `pr-downloader` are on your PATH or referenced by absolute path, then — using the same `GAME`/`MAP` values as the `env:` block in `.github/workflows/synctest.yml`: + +``` +GAME="Beyond All Reason test-30212-63c6ded" +MAP="Jade Empress 1.41" + +# The PRD_* vars point at BAR's CDN; without them pr-downloader resolves a +# different rapid master than CI does. +PRD_RAPID_USE_STREAMER=false \ +PRD_RAPID_REPO_MASTER=https://repos-cdn.beyondallreason.dev/repos.gz \ +PRD_HTTP_SEARCH_URL=https://files-cdn.beyondallreason.dev/find \ +pr-downloader --filesystem-writepath ./bar-data \ + --download-game "$GAME" \ + --download-map "$MAP" + +# synctest-startscript.txt is a template — render the pins into it first. +sed -e "s/@VERSION@/${GAME##* }/g" -e "s/@MAPNAME@/$MAP/g" \ + test/synctest/synctest-startscript.txt > startscript.txt + +spring-headless --isolation --write-dir ./bar-data startscript.txt +``` + +A successful run exits 0 at game frame ~2100 (the startscript's trailing `quitforce`) and leaves `bar-data/synctest_synchash.json` — a JSON file whose `digest` field is what CI compares. + +## Bumping the pinned BAR version + +BAR's rapid `test--` tags are content-addressed — once published they never change — which is why the CI doesn't need an external version manifest. The only place to bump is the `GAME:` env in `.github/workflows/synctest.yml` (and `MAP:` for the map): the workflow feeds them to `pr-downloader`, substitutes them into the startscript template's `gametype=Beyond All Reason @VERSION@;` and `mapname=@MAPNAME@;` lines, and uses both in the cache key. + + +## Replacing the startscript + +The startscript was generated by BAR from running a fightertest scenario in the actual BAR lobby, then pulled from the bar data dir; modifying it to instead run synctest on a different map. If it ever needs changing, that's the easiest way to regenerate it. + +A trailing `quitforce` in the startscript's `debugcommands` is required so the engine actually quits when the test finishes. diff --git a/test/synctest/synctest-startscript.txt b/test/synctest/synctest-startscript.txt new file mode 100644 index 0000000000..ce644af5d6 --- /dev/null +++ b/test/synctest/synctest-startscript.txt @@ -0,0 +1,57 @@ +[game] +{ + [allyteam1] + { + startrectleft=0.82999998; + startrectright=1; + startrectbottom=1; + numallies=0; + startrecttop=0; + } + [team0] + { + startposx=800; + allyteam=0; + teamleader=0; + side=Armada; + startposz=100; + } + [allyteam0] + { + startrectleft=0; + startrectright=0.17; + startrectbottom=1; + numallies=0; + startrecttop=0; + } + [modoptions] + { + debugcommands=2:cheat|3:globallos|15:luarules synctest 2000 0.33 0.33 0.1 0.5|20:setspeed 20|21:speedcontrol 0|2100:quitforce; + } + [player0] + { + team=0; + name=Player; + } + [ai0] + { + team=1; + host=0; + shortname=NullAI; + } + [team1] + { + allyteam=1; + teamleader=0; + side=Cortex; + startposx=100; + startposz=800; + } + ishost=1; + startpostype=3; + gametype=Beyond All Reason @VERSION@; + mapname=@MAPNAME@; + myplayername=Player; + fixedrngseed=123123; + numplayers=1; +}