From 174f8b1a9efff20d9cb82d62da08cee76196bd21 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 15:14:04 +0000 Subject: [PATCH 01/14] Use NuGet for vcpkg binary cache --- .github/workflows/build.yaml | 3 +++ .github/workflows/build_base.yaml | 33 +++++++++++++++++-------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d06959871c..39fb155218 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,9 @@ jobs: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml needs: [platform-cache] + permissions: + packages: write + contents: read secrets: GH_TOKEN_DEV: ${{ secrets.GH_TOKEN_DEV }} with: diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index c52e107aa5..1d5a65594d 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -16,7 +16,7 @@ on: required: true permissions: - packages: read + packages: write contents: read defaults: @@ -41,6 +41,7 @@ jobs: CCACHE_MAXSIZE: "10G" CCACHE_COMPRESS: "true" CCACHE_COMPRESSLEVEL: "6" + VCPKG_NUGET_FEED_URL: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json steps: - name: Allow safe directories run: git config --global --add safe.directory '*' @@ -49,14 +50,6 @@ jobs: with: submodules: recursive - - name: Restore vcpkg binary cache - uses: actions/cache@v5 - with: - path: ${{ github.workspace }}/vcpkg-binary-cache - key: vcpkg-binaries-${{ matrix.platform }}-${{ hashFiles('vcpkg.json') }} - restore-keys: | - vcpkg-binaries-${{ matrix.platform }}- - - name: Restore ccache uses: actions/cache@v5 with: @@ -71,19 +64,29 @@ jobs: echo "Building for ${{ matrix.platform }}" # Use $GITHUB_WORKSPACE (container path /__w/...) not ${{ github.workspace }} - # (host path /home/runner/work/...) so ccache/vcpkg write to the mounted - # volume and persist for actions/cache to save. + # (host path /home/runner/work/...) so ccache writes to the mounted + # volume and persists for actions/cache to save. export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" - export VCPKG_BINARY_SOURCES="clear;files,$GITHUB_WORKSPACE/vcpkg-binary-cache,readwrite" + export VCPKG_EXE="$GITHUB_WORKSPACE/vcpkg/vcpkg" + export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED_URL,readwrite" - # Clean intermediate vcpkg artifacts but preserve binary cache and downloads + # Clean intermediate vcpkg artifacts but preserve downloads rm -rf vcpkg/buildtrees vcpkg/packages vcpkg/vcpkg_installed \ build/vcpkg_installed ~/.cache/vcpkg ~/.vcpkg - mkdir -p "$GITHUB_WORKSPACE/vcpkg-binary-cache" - ./vcpkg/bootstrap-vcpkg.sh + NUGET_EXE="$($VCPKG_EXE fetch nuget | tail -n 1)" + mono "$NUGET_EXE" sources remove -Name GitHubPackages || true + mono "$NUGET_EXE" sources add \ + -Source "$VCPKG_NUGET_FEED_URL" \ + -StorePasswordInClearText \ + -Name GitHubPackages \ + -UserName "${{ github.repository_owner }}" \ + -Password "${{ secrets.GH_TOKEN_DEV }}" + mono "$NUGET_EXE" setapikey "${{ secrets.GH_TOKEN_DEV }}" \ + -Source "$VCPKG_NUGET_FEED_URL" + chown -R $(id -u):$(id -g) $PWD # Reset ccache stats for this build From 876dce173114f220bde212a8ed579a6c97aede6d Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 15:25:08 +0000 Subject: [PATCH 02/14] Install Mono for vcpkg NuGet cache --- .cicd/platforms/asan.Dockerfile | 3 ++- .cicd/platforms/asserton.Dockerfile | 3 ++- .cicd/platforms/gcc.Dockerfile | 3 ++- .cicd/platforms/ubsan.Dockerfile | 3 ++- .cicd/platforms/ubuntu24.Dockerfile | 3 ++- .github/workflows/build_base.yaml | 1 + 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.cicd/platforms/asan.Dockerfile b/.cicd/platforms/asan.Dockerfile index e431ce247b..fa3145945f 100644 --- a/.cicd/platforms/asan.Dockerfile +++ b/.cicd/platforms/asan.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -75,4 +76,4 @@ COPY <<-EOF /extras.cmake EOF ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 -ENV ASAN_OPTIONS=detect_leaks=0 \ No newline at end of file +ENV ASAN_OPTIONS=detect_leaks=0 diff --git a/.cicd/platforms/asserton.Dockerfile b/.cicd/platforms/asserton.Dockerfile index f39066d1fa..8884d48442 100644 --- a/.cicd/platforms/asserton.Dockerfile +++ b/.cicd/platforms/asserton.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -78,4 +79,4 @@ COPY <<-EOF /extras.cmake set(SYSIO_ENABLE_RELEASE_BUILD_TEST "Off" CACHE BOOL "") EOF -ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 \ No newline at end of file +ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 diff --git a/.cicd/platforms/gcc.Dockerfile b/.cicd/platforms/gcc.Dockerfile index e2ffc30fbb..3f260f0d8f 100644 --- a/.cicd/platforms/gcc.Dockerfile +++ b/.cicd/platforms/gcc.Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ sudo \ golang \ python3-dev \ @@ -62,4 +63,4 @@ COPY <<-EOF /extras.cmake set(CMAKE_CXX_FLAGS "-Wall -Wextra -fdiagnostics-color=always" CACHE STRING "") EOF -ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 \ No newline at end of file +ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 diff --git a/.cicd/platforms/ubsan.Dockerfile b/.cicd/platforms/ubsan.Dockerfile index b69c82e863..8738b1a850 100644 --- a/.cicd/platforms/ubsan.Dockerfile +++ b/.cicd/platforms/ubsan.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -79,4 +80,4 @@ COPY <<-EOF /extras.cmake EOF ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 -ENV UBSAN_OPTIONS=print_stacktrace=1,suppressions=/ubsan.supp \ No newline at end of file +ENV UBSAN_OPTIONS=print_stacktrace=1,suppressions=/ubsan.supp diff --git a/.cicd/platforms/ubuntu24.Dockerfile b/.cicd/platforms/ubuntu24.Dockerfile index 4504b80c0b..bef6769697 100644 --- a/.cicd/platforms/ubuntu24.Dockerfile +++ b/.cicd/platforms/ubuntu24.Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ sudo \ golang \ python3-dev \ @@ -52,4 +53,4 @@ RUN apt-get update && apt-get upgrade -y && \ ENV CC=/usr/bin/clang-18 ENV CXX=/usr/bin/clang++-18 -ENV CMAKE_MAKE_PROGRAM=/usr/bin/ninja \ No newline at end of file +ENV CMAKE_MAKE_PROGRAM=/usr/bin/ninja diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 1d5a65594d..0ed6dec815 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -41,6 +41,7 @@ jobs: CCACHE_MAXSIZE: "10G" CCACHE_COMPRESS: "true" CCACHE_COMPRESSLEVEL: "6" + VCPKG_DISABLE_METRICS: "1" VCPKG_NUGET_FEED_URL: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json steps: - name: Allow safe directories From 00ed5f79ee3a6e845de6539208a5b58776170dde Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 18:33:46 +0000 Subject: [PATCH 03/14] Share vcpkg build configuration script --- .github/workflows/build_base.yaml | 43 ++-- BUILD.md | 167 +++++++++++++--- scripts/build-with-github-vcpkg-cache.sh | 244 +++++++++++++++++++++++ 3 files changed, 391 insertions(+), 63 deletions(-) create mode 100755 scripts/build-with-github-vcpkg-cache.sh diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 0ed6dec815..29d96ff48e 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -43,6 +43,7 @@ jobs: CCACHE_COMPRESSLEVEL: "6" VCPKG_DISABLE_METRICS: "1" VCPKG_NUGET_FEED_URL: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json + VCPKG_NUGET_USER: ${{ github.repository_owner }} steps: - name: Allow safe directories run: git config --global --add safe.directory '*' @@ -61,6 +62,9 @@ jobs: - name: Build id: build + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_DEV }} + GITHUB_USER: ${{ github.repository_owner }} run: | echo "Building for ${{ matrix.platform }}" @@ -68,48 +72,25 @@ jobs: # (host path /home/runner/work/...) so ccache writes to the mounted # volume and persists for actions/cache to save. export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" - export VCPKG_EXE="$GITHUB_WORKSPACE/vcpkg/vcpkg" - export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED_URL,readwrite" - - # Clean intermediate vcpkg artifacts but preserve downloads - rm -rf vcpkg/buildtrees vcpkg/packages vcpkg/vcpkg_installed \ - build/vcpkg_installed ~/.cache/vcpkg ~/.vcpkg - - ./vcpkg/bootstrap-vcpkg.sh - - NUGET_EXE="$($VCPKG_EXE fetch nuget | tail -n 1)" - mono "$NUGET_EXE" sources remove -Name GitHubPackages || true - mono "$NUGET_EXE" sources add \ - -Source "$VCPKG_NUGET_FEED_URL" \ - -StorePasswordInClearText \ - -Name GitHubPackages \ - -UserName "${{ github.repository_owner }}" \ - -Password "${{ secrets.GH_TOKEN_DEV }}" - mono "$NUGET_EXE" setapikey "${{ secrets.GH_TOKEN_DEV }}" \ - -Source "$VCPKG_NUGET_FEED_URL" chown -R $(id -u):$(id -g) $PWD # Reset ccache stats for this build ccache -z || true - cmake -B build -S . -G Ninja ${SYSIO_PLATFORM_HAS_EXTRAS_CMAKE:+-C /extras.cmake} \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_CXX_COMPILER=$CXX \ - -DCMAKE_MAKE_PROGRAM=$CMAKE_MAKE_PROGRAM \ - -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_CCACHE=ON \ - -DENABLE_TESTS=ON \ - ${CMAKE_PREFIX_PATH:+-DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH} - if [[ "${{ matrix.platform }}" == "gcc" ]]; then echo "Using reduced parallelism for gcc platform" - cmake --build build -- -j 8 + BUILD_JOBS=8 else - cmake --build build -- -j 11 + BUILD_JOBS=11 fi + scripts/build-with-github-vcpkg-cache.sh \ + --mode trusted-ci \ + --build-dir build \ + --jobs "$BUILD_JOBS" \ + --skip-tests + echo "=== ccache statistics ===" ccache -s || true diff --git a/BUILD.md b/BUILD.md index 99eb7510d1..63304112fd 100644 --- a/BUILD.md +++ b/BUILD.md @@ -135,57 +135,151 @@ This will build the vcpkg executable and set up the local vcpkg infrastructure. You are now ready to build Wire Sysio. -## Step 2 - Build +## Recommended: Build with the GitHub Packages vcpkg Cache -Make sure you are in the root of the `wire-sysio` repo, then perform the build with CMake and your chosen compiler. +Make sure you are in the root of the `wire-sysio` repo, then perform the build with the shared script used by CI and local developer builds. > ⚠️ **Memory/Parallelism Warning** ⚠️ > Building Wire Sysio from source can be resource-intensive. Some source files require **up to 4 GB of RAM** each to compile. If you use all CPU cores for parallel compilation (e.g. `make -j$(nproc)` or Ninja default parallelism), you may exhaust memory and encounter compiler crashes. Consider using a lower parallel job count (`-j`) if you run into memory issues or if you need to use your machine for other tasks during the build. -### Build Instructions - -First, ensure the environment is set to use **Clang 18** as the compiler: +The simplest way to build with the same vcpkg NuGet binary cache used by CI is +to run: ```bash -# Example: set CC and CXX to Clang 18 compilers (adjust path if installed elsewhere) export CC=/opt/clang/clang-18/bin/clang export CXX=/opt/clang/clang-18/bin/clang++ +export CMAKE_PREFIX_PATH="/opt/clang/clang-18" + +scripts/build-with-github-vcpkg-cache.sh +``` + +The script bootstraps vcpkg, configures the GitHub Packages NuGet binary cache, +runs CMake with the repository vcpkg toolchain, builds the project, and runs +tests. + +When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` +and stores cache files in `.ccache` by default. + +Useful options: + +```bash +scripts/build-with-github-vcpkg-cache.sh --build-dir build/release +scripts/build-with-github-vcpkg-cache.sh --jobs 8 +scripts/build-with-github-vcpkg-cache.sh --skip-tests +``` + +The script has three build modes: + +- `developer`: local developer builds; reads packages from the GitHub Packages + NuGet cache and never publishes packages +- `trusted-ci`: trusted GitHub Actions runs; reads and writes the GitHub + Packages NuGet cache +- `forked-pr-ci`: fork pull-request runs; uses vcpkg's default local cache so + the workflow does not need package credentials + +The default mode is `developer`. The GitHub Actions workflow uses the same +script with `--mode trusted-ci --skip-tests`, because CI runs tests in separate +jobs after archiving the build directory. + +## Optional: Use the GitHub Packages vcpkg Binary Cache Manually + +The project can restore vcpkg-built dependencies from the same NuGet-backed +binary cache used by CI. This is optional, but it avoids rebuilding large vcpkg +dependencies locally. + +To use the cache, you need: + +- a GitHub token that can read Wire-Network GitHub Packages +- `read:packages` scope on that token +- Mono, because vcpkg runs `nuget.exe` on Linux + +Install Mono: + +```bash +sudo apt-get install -y mono-complete +``` + +If you use the GitHub CLI, refresh the local token with package-read scope: + +```bash +gh auth refresh -h github.com -s read:packages +gh auth status +``` + +`gh auth status` should list `read:packages` in the token scopes. + +Configure the NuGet source: + +```bash +export GITHUB_TOKEN="$(gh auth token)" +export GITHUB_USER="$(gh api user --jq .login)" +export VCPKG_NUGET_FEED="https://nuget.pkg.github.com/Wire-Network/index.json" + +NUGET_EXE="$(./vcpkg/vcpkg fetch nuget | tail -n 1)" + +mono "$NUGET_EXE" sources remove -Name "github" >/dev/null 2>&1 || true +mono "$NUGET_EXE" sources add \ + -Name "github" \ + -Source "$VCPKG_NUGET_FEED" \ + -UserName "$GITHUB_USER" \ + -Password "$GITHUB_TOKEN" \ + -StorePasswordInClearText + +mono "$NUGET_EXE" setapikey "$GITHUB_TOKEN" -Source "$VCPKG_NUGET_FEED" ``` -Now run CMake to configure the build and generate build files. We recommend using **Ninja** as the build generator for faster builds (we installed `ninja-build` earlier): +Enable read-only binary cache restores for the current shell: + +```bash +export VCPKG_FEATURE_FLAGS="manifests,binarycaching" +export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,read" +``` + +A successful restore looks like: + +```text +Restored 9 package(s) from NuGet +``` + +If vcpkg prints `Restored 0 package(s) from NuGet`, check: + +- `gh auth status` includes `read:packages` +- the compiler and platform match the CI platform image +- the vcpkg manifest, registry configuration, and dependency features match CI + +## Configure + +From the repository root: ```bash +export CC=/opt/clang/clang-18/bin/clang +export CXX=/opt/clang/clang-18/bin/clang++ +export CMAKE_MAKE_PROGRAM=/usr/bin/ninja +export CMAKE_PREFIX_PATH="/opt/clang/clang-18" + cmake -B build -S . -G Ninja \ -DCMAKE_C_COMPILER="$CC" \ -DCMAKE_CXX_COMPILER="$CXX" \ + -DCMAKE_MAKE_PROGRAM="$CMAKE_MAKE_PROGRAM" \ -DCMAKE_TOOLCHAIN_FILE="$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="/opt/clang/clang-18" \ - -DENABLE_CCACHE=ON + -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ + -DENABLE_CCACHE=ON \ + -DENABLE_TESTS=ON ``` -In the above command: -- `-B build -S .` tells CMake to create (or use) the directory `build/` for the build files. -- `-G Ninja` chooses the Ninja generator. If you prefer Makefiles, you can omit this (default is Unix Makefiles), but ensure you adjust build commands accordingly. -- `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` are pointed to Clang 18 (from Step 1b). -- `CMAKE_TOOLCHAIN_FILE` points to the vcpkg toolchain file, so CMake will integrate vcpkg dependencies automatically. -- `CMAKE_BUILD_TYPE=Release` produces an optimized build. (You can use `Debug` for development, etc.) -- `CMAKE_PREFIX_PATH` is set to install location where you plan to install the CDT tooling (if not the system prefix `/usr`). -- `ENABLE_CCACHE=ON` will use **ccache** to cache compilation results (if `ccache` is installed). This can significantly speed up rebuilds. You can omit this or set `OFF` if you do not have ccache or do not want to use it. - -If CMake configuration is successful, it will generate the build files in the `build/` directory. Now proceed to compile: +## Build ```bash -cmake --build build -- -j$(nproc) +cmake --build build -- -j "$(nproc)" ``` -This will start the build process (using all available CPU cores by default). If you find your system running low on memory or becoming unresponsive, cancel the build (`Ctrl+C`) and re-run with a lower parallel job count, for example: +If you find your system running low on memory or becoming unresponsive, cancel +the build (`Ctrl+C`) and re-run with a lower parallel job count, for example: ```bash -cmake --build build -- -j4 -``` - -(to limit to 4 threads). Ninja will respect the `-j` flag similarly, or you can set the environment variable `CMAKE_BUILD_PARALLEL_LEVEL` before running the `cmake --build` command. +cmake --build build -- -j 4 +``` Once the build completes successfully, the Wire Sysio binaries and libraries will be available in `build/bin/` (and other subdirectories under `build/`). @@ -225,7 +319,7 @@ Choose **either** method A or B according to your preference. Method A (deb pack Wire Sysio also provides a Docker-based build environment for convenience. This is the easiest way to build and run Wire Sysio without manually installing all dependencies, as the Dockerfile encapsulates all requirements (including Clang 18, etc.) and build steps. -The provided Docker setup supports building on any modern 64-bit Linux or Apple Silicon (arm64) host that can run Docker (x86_64, arm64/aarch64). The container will produce Wire Sysio binaries for the same architecture as the host. +The provided Docker setup is intended for x86_64 hosts that can run Docker. The Dockerfile configures vcpkg with the `x64-linux` triplet. To build Wire Sysio using Docker: @@ -234,24 +328,33 @@ To build Wire Sysio using Docker: ./scripts/docker-build.sh ``` -This script will build the Docker image (using the `./etc/docker/Dockerfile`) and compile Wire Sysio inside it. The final Wire Sysio build artifacts will be available within the Docker image (and can be copied out or used via Docker container). +This script will build the Docker image using `etc/docker/Dockerfile` and compile Wire Sysio inside it. The final Wire Sysio build artifacts are available inside the image under `/wire/wire-sysio/build/Release`. -By default, it builds the `app-build-repo` target (fetching the source from the repository). You can also build from your local source or choose other stages: +By default, it builds the `app-build-local` target using your current checkout as the source context. You can also choose other stages: -- To build using **local source context** (your cloned repository code), use the `app-build-local` target: +- To build using **local source context** explicitly, use the `app-build-local` target: ```bash ./scripts/docker-build.sh --target=app-build-local ``` -- To specify a different Git branch or repository for the source, set the `SYSIO_BRANCH` or `REPO` build args via the script (see script help for usage). +- To build from the remote repository source, use the `app-build-repo` target: + ```bash + ./scripts/docker-build.sh --target=app-build-repo --sysio-branch=master + ``` +- To build the repository source with Wire CDT as well, use the `cdt-build-repo` target: + ```bash + ./scripts/docker-build.sh --target=cdt-build-repo --sysio-branch=master --cdt-branch=master + ``` -- To tag the resulting Docker image with a specific name (default tag is `wire/sysio:latest`), use the `--tag` option: +- To tag the resulting Docker image with a specific name (default tag is `wire/sysio`), use the `--tag` option: ```bash ./scripts/docker-build.sh --target=app-build-local --tag=wire/sysio:mybuild ``` -- There is an additional flag that can be used to specify which Ubuntu version your image builds on (defaults Ubuntu 24.04) +- To specify which Ubuntu base image tag to build on, use `--ubuntu-tag` (defaults to `24.04`): ```bash ./scripts/docker-build.sh --ubuntu-tag=24.04 ``` + +The script requires at least 32 GiB of available memory and passes `--memory 32G` to `docker build`. After the script finishes, you will have a Docker image with Wire Sysio built inside. You can run this image or extract the built binaries. (Refer to the Dockerfile targets and the script for details on where the binaries reside in each stage.) diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh new file mode 100755 index 0000000000..8835cb5147 --- /dev/null +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" +JOBS="${JOBS:-$(nproc)}" +RUN_TESTS=1 +NEEDS_CI_OWNERSHIP_FIX=0 +BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" +VCPKG_NUGET_FEED="${VCPKG_NUGET_FEED:-${VCPKG_NUGET_FEED_URL:-https://nuget.pkg.github.com/Wire-Network/index.json}}" + +usage() { + cat <&2 + exit 1 +} + +info() { + printf '==> %s\n' "$*" +} + +require_command() { + local command_name="$1" + local package_hint="$2" + + if ! command -v "$command_name" >/dev/null 2>&1; then + fail "'$command_name' is not installed." "Install it with:\n sudo apt-get install -y $package_hint" + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --build-dir) + [[ $# -ge 2 ]] || fail "--build-dir requires a value." "Run '$0 --build-dir /path/to/build'." + BUILD_DIR="$2" + shift 2 + ;; + --jobs) + [[ $# -ge 2 ]] || fail "--jobs requires a value." "Run '$0 --jobs $(nproc)'." + JOBS="$2" + shift 2 + ;; + --skip-tests) + RUN_TESTS=0 + shift + ;; + --mode) + [[ $# -ge 2 ]] || fail "--mode requires a value." "Use '--mode developer', '--mode trusted-ci', or '--mode forked-pr-ci'." + BUILD_MODE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "Unknown option '$1'." "Run '$0 --help' to see supported options." + ;; + esac +done + +if [[ "$BUILD_MODE" != "developer" && "$BUILD_MODE" != "trusted-ci" && "$BUILD_MODE" != "forked-pr-ci" ]]; then + fail "Unsupported build mode '$BUILD_MODE'." "Use '--mode developer' for local builds, '--mode trusted-ci' for trusted GitHub Actions runs, or '--mode forked-pr-ci' for fork pull requests." +fi + +if [[ "$BUILD_MODE" == "trusted-ci" || "$BUILD_MODE" == "forked-pr-ci" ]]; then + NEEDS_CI_OWNERSHIP_FIX=1 +fi + +if [[ ! -f "$ROOT_DIR/CMakeLists.txt" || ! -d "$ROOT_DIR/vcpkg" ]]; then + fail "The script could not locate the Wire Sysio repository root." "Run this script from a complete Wire Sysio checkout with submodules initialized:\n git submodule update --init --recursive" +fi + +require_command cmake cmake +require_command ninja ninja-build +require_command git git + +if [[ "$BUILD_MODE" != "forked-pr-ci" ]]; then + require_command mono mono-complete +fi + +if [[ "$BUILD_MODE" == "developer" ]]; then + require_command gh gh +fi + +if [[ "$NEEDS_CI_OWNERSHIP_FIX" -eq 1 ]]; then + chown -R "$(id -u):$(id -g)" "$ROOT_DIR" +fi + +rm -rf "$ROOT_DIR/vcpkg/buildtrees" "$ROOT_DIR/vcpkg/packages" "$ROOT_DIR/vcpkg/vcpkg_installed" \ + "$BUILD_DIR/vcpkg_installed" ~/.cache/vcpkg ~/.vcpkg + +if [[ ! -x "$ROOT_DIR/vcpkg/vcpkg" ]]; then + info "Bootstrapping vcpkg" + "$ROOT_DIR/vcpkg/bootstrap-vcpkg.sh" +else + "$ROOT_DIR/vcpkg/bootstrap-vcpkg.sh" +fi + +export VCPKG_DISABLE_METRICS="${VCPKG_DISABLE_METRICS:-1}" + +if [[ "$BUILD_MODE" == "forked-pr-ci" ]]; then + export VCPKG_BINARY_SOURCES="clear;default,readwrite" +else + if [[ "$BUILD_MODE" == "trusted-ci" ]]; then + GITHUB_TOKEN="${GITHUB_TOKEN:-${VCPKG_NUGET_TOKEN:-}}" + GITHUB_USER="${GITHUB_USER:-${VCPKG_NUGET_USER:-}}" + + if [[ -z "$GITHUB_TOKEN" || -z "$GITHUB_USER" ]]; then + fail "trusted-ci mode requires GITHUB_TOKEN and GITHUB_USER." "In the workflow step, pass:\n GITHUB_TOKEN: \${{ secrets.GH_TOKEN_DEV }}\n GITHUB_USER: \${{ github.repository_owner }}\nFor fork PRs, use '--mode forked-pr-ci' instead." + fi + else + if ! gh auth status -h github.com >/dev/null 2>&1; then + fail "GitHub CLI is not authenticated." "Authenticate and request package-read scope:\n gh auth login -h github.com\n gh auth refresh -h github.com -s read:packages" + fi + + GH_STATUS="$(gh auth status -h github.com 2>&1 || true)" + if [[ "$GH_STATUS" != *"read:packages"* ]]; then + fail "GitHub CLI token is missing the 'read:packages' scope." "Refresh the token scope, then rerun this script:\n gh auth refresh -h github.com -s read:packages\n gh auth status -h github.com" + fi + + GITHUB_TOKEN="${GITHUB_TOKEN:-$(gh auth token)}" + GITHUB_USER="${GITHUB_USER:-$(gh api user --jq .login)}" + + if [[ -z "$GITHUB_TOKEN" || -z "$GITHUB_USER" ]]; then + fail "Could not resolve GitHub token or username." "Set GITHUB_TOKEN and GITHUB_USER explicitly, or fix GitHub CLI authentication with 'gh auth login'." + fi + fi + + NUGET_EXE="$("$ROOT_DIR/vcpkg/vcpkg" fetch nuget | tail -n 1)" + if [[ ! -f "$NUGET_EXE" ]]; then + fail "vcpkg did not return a usable nuget.exe path." "Run '$ROOT_DIR/vcpkg/vcpkg fetch nuget' and fix any reported vcpkg download errors." + fi + + info "Configuring GitHub Packages NuGet source" + mono "$NUGET_EXE" sources remove -Name "github" >/dev/null 2>&1 || true + mono "$NUGET_EXE" sources add \ + -Name "github" \ + -Source "$VCPKG_NUGET_FEED" \ + -UserName "$GITHUB_USER" \ + -Password "$GITHUB_TOKEN" \ + -StorePasswordInClearText >/dev/null + mono "$NUGET_EXE" setapikey "$GITHUB_TOKEN" -Source "$VCPKG_NUGET_FEED" >/dev/null + + if [[ "$BUILD_MODE" == "developer" ]]; then + export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,read" + else + export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,readwrite" + fi +fi + +export CC="${CC:-/usr/bin/clang-18}" +export CXX="${CXX:-/usr/bin/clang++-18}" +export CMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM:-/usr/bin/ninja}" +export CCACHE_DIR="${CCACHE_DIR:-$ROOT_DIR/.ccache}" +export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-10G}" + +CMAKE_ARGS=( + -B "$BUILD_DIR" + -S "$ROOT_DIR" + -G Ninja +) + +if [[ -n "${SYSIO_PLATFORM_HAS_EXTRAS_CMAKE:-}" ]]; then + CMAKE_ARGS+=(-C /extras.cmake) +fi + +CMAKE_ARGS+=( + -DCMAKE_C_COMPILER="$CC" + -DCMAKE_CXX_COMPILER="$CXX" + -DCMAKE_MAKE_PROGRAM="$CMAKE_MAKE_PROGRAM" + -DCMAKE_TOOLCHAIN_FILE="$ROOT_DIR/vcpkg/scripts/buildsystems/vcpkg.cmake" + -DCMAKE_BUILD_TYPE=Release + -DENABLE_CCACHE=ON + -DENABLE_TESTS=ON +) + +if [[ -n "${CMAKE_PREFIX_PATH:-}" ]]; then + CMAKE_ARGS+=(-DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH") +fi + +CONFIGURE_LOG="$BUILD_DIR/vcpkg-nuget-configure.log" +mkdir -p "$BUILD_DIR" + +info "Build directory: $BUILD_DIR" +info "NuGet feed: $VCPKG_NUGET_FEED" +info "Build mode: $BUILD_MODE" +info "vcpkg binary sources: $VCPKG_BINARY_SOURCES" + +info "Configuring CMake" +set +e +cmake "${CMAKE_ARGS[@]}" 2>&1 | tee "$CONFIGURE_LOG" +configure_status=${PIPESTATUS[0]} +set -e + +if [[ "$configure_status" -ne 0 ]]; then + fail "CMake configure failed." "Review $CONFIGURE_LOG. Common fixes:\n sudo apt-get install -y mono-complete ninja-build cmake\n gh auth refresh -h github.com -s read:packages\n rm -rf '$BUILD_DIR' and rerun this script after changing compilers." +fi + +if grep -q "Restored 0 package(s) from NuGet" "$CONFIGURE_LOG"; then + info "Warning: vcpkg reported zero NuGet restores. This can mean the ABI does not match CI, or that the packages were already installed in '$BUILD_DIR'." +fi + +if grep -q "Restored [1-9][0-9]* package(s) from NuGet" "$CONFIGURE_LOG"; then + info "Confirmed vcpkg restored packages from the GitHub NuGet cache" +else + info "No NuGet restore line was printed. This usually means vcpkg packages were already installed in '$BUILD_DIR'." +fi + +info "Building" +cmake --build "$BUILD_DIR" -- -j "$JOBS" + +if [[ "$RUN_TESTS" -eq 1 ]]; then + info "Running tests" + ctest --test-dir "$BUILD_DIR" -j "$JOBS" --output-on-failure +fi + +info "Done" From 98731fc56d21a185d2b910a8432446b6c1f74b2b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 21:09:00 +0000 Subject: [PATCH 04/14] Address vcpkg cache script review --- .github/workflows/build_base.yaml | 10 +++++- BUILD.md | 9 ++++-- scripts/build-with-github-vcpkg-cache.sh | 39 ++++++++++++++++-------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 29d96ff48e..da8bc93330 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -65,6 +65,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_DEV }} GITHUB_USER: ${{ github.repository_owner }} + IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} run: | echo "Building for ${{ matrix.platform }}" @@ -85,8 +86,15 @@ jobs: BUILD_JOBS=11 fi + BUILD_MODE=trusted-ci + if [[ "$IS_FORK_PR" == "true" ]]; then + echo "Using local vcpkg binary cache for fork pull request" + BUILD_MODE=forked-pr-ci + unset GITHUB_TOKEN GITHUB_USER + fi + scripts/build-with-github-vcpkg-cache.sh \ - --mode trusted-ci \ + --mode "$BUILD_MODE" \ --build-dir build \ --jobs "$BUILD_JOBS" \ --skip-tests diff --git a/BUILD.md b/BUILD.md index 63304112fd..dc01fbcf8f 100644 --- a/BUILD.md +++ b/BUILD.md @@ -165,6 +165,7 @@ Useful options: ```bash scripts/build-with-github-vcpkg-cache.sh --build-dir build/release scripts/build-with-github-vcpkg-cache.sh --jobs 8 +scripts/build-with-github-vcpkg-cache.sh --clean scripts/build-with-github-vcpkg-cache.sh --skip-tests ``` @@ -177,9 +178,11 @@ The script has three build modes: - `forked-pr-ci`: fork pull-request runs; uses vcpkg's default local cache so the workflow does not need package credentials -The default mode is `developer`. The GitHub Actions workflow uses the same -script with `--mode trusted-ci --skip-tests`, because CI runs tests in separate -jobs after archiving the build directory. +The default mode is `developer`. Developer mode keeps the local vcpkg caches +between runs unless `--clean` is passed. GitHub Actions uses the same script +with `--skip-tests`; trusted pull requests run in `trusted-ci` mode, while fork +pull requests run in `forked-pr-ci` mode. CI runs tests in separate jobs after +archiving the build directory. ## Optional: Use the GitHub Packages vcpkg Binary Cache Manually diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index 8835cb5147..c3db7642be 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -6,6 +6,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" JOBS="${JOBS:-$(nproc)}" RUN_TESTS=1 +CLEAN="${WIRE_SYSIO_CLEAN_BUILD:-}" NEEDS_CI_OWNERSHIP_FIX=0 BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" VCPKG_NUGET_FEED="${VCPKG_NUGET_FEED:-${VCPKG_NUGET_FEED_URL:-https://nuget.pkg.github.com/Wire-Network/index.json}}" @@ -19,6 +20,8 @@ Build Wire Sysio with the GitHub Packages vcpkg NuGet binary cache. Options: --build-dir DIR CMake build directory. Default: $BUILD_DIR --jobs N Parallel build jobs. Default: $JOBS + --clean Remove vcpkg build artifacts before configuring. + Default: enabled in CI modes, disabled in developer mode. --skip-tests Configure and build only. --mode MODE Build mode: developer, trusted-ci, or forked-pr-ci. Default: $BUILD_MODE @@ -66,6 +69,10 @@ while [[ $# -gt 0 ]]; do JOBS="$2" shift 2 ;; + --clean) + CLEAN=1 + shift + ;; --skip-tests) RUN_TESTS=0 shift @@ -89,6 +96,14 @@ if [[ "$BUILD_MODE" != "developer" && "$BUILD_MODE" != "trusted-ci" && "$BUILD_M fail "Unsupported build mode '$BUILD_MODE'." "Use '--mode developer' for local builds, '--mode trusted-ci' for trusted GitHub Actions runs, or '--mode forked-pr-ci' for fork pull requests." fi +if [[ -z "$CLEAN" ]]; then + if [[ "$BUILD_MODE" == "developer" ]]; then + CLEAN=0 + else + CLEAN=1 + fi +fi + if [[ "$BUILD_MODE" == "trusted-ci" || "$BUILD_MODE" == "forked-pr-ci" ]]; then NEEDS_CI_OWNERSHIP_FIX=1 fi @@ -113,16 +128,14 @@ if [[ "$NEEDS_CI_OWNERSHIP_FIX" -eq 1 ]]; then chown -R "$(id -u):$(id -g)" "$ROOT_DIR" fi -rm -rf "$ROOT_DIR/vcpkg/buildtrees" "$ROOT_DIR/vcpkg/packages" "$ROOT_DIR/vcpkg/vcpkg_installed" \ - "$BUILD_DIR/vcpkg_installed" ~/.cache/vcpkg ~/.vcpkg - -if [[ ! -x "$ROOT_DIR/vcpkg/vcpkg" ]]; then - info "Bootstrapping vcpkg" - "$ROOT_DIR/vcpkg/bootstrap-vcpkg.sh" -else - "$ROOT_DIR/vcpkg/bootstrap-vcpkg.sh" +if [[ "$CLEAN" == "1" ]]; then + rm -rf "$ROOT_DIR/vcpkg/buildtrees" "$ROOT_DIR/vcpkg/packages" "$ROOT_DIR/vcpkg/vcpkg_installed" \ + "$BUILD_DIR/vcpkg_installed" ~/.cache/vcpkg ~/.vcpkg fi +info "Bootstrapping vcpkg" +"$ROOT_DIR/vcpkg/bootstrap-vcpkg.sh" + export VCPKG_DISABLE_METRICS="${VCPKG_DISABLE_METRICS:-1}" if [[ "$BUILD_MODE" == "forked-pr-ci" ]]; then @@ -160,6 +173,7 @@ else info "Configuring GitHub Packages NuGet source" mono "$NUGET_EXE" sources remove -Name "github" >/dev/null 2>&1 || true + # NuGet on Linux under Mono needs this flag to persist the feed credential. mono "$NUGET_EXE" sources add \ -Name "github" \ -Source "$VCPKG_NUGET_FEED" \ @@ -205,7 +219,7 @@ if [[ -n "${CMAKE_PREFIX_PATH:-}" ]]; then CMAKE_ARGS+=(-DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH") fi -CONFIGURE_LOG="$BUILD_DIR/vcpkg-nuget-configure.log" +CONFIGURE_LOG="$BUILD_DIR/cmake-configure.log" mkdir -p "$BUILD_DIR" info "Build directory: $BUILD_DIR" @@ -225,9 +239,7 @@ fi if grep -q "Restored 0 package(s) from NuGet" "$CONFIGURE_LOG"; then info "Warning: vcpkg reported zero NuGet restores. This can mean the ABI does not match CI, or that the packages were already installed in '$BUILD_DIR'." -fi - -if grep -q "Restored [1-9][0-9]* package(s) from NuGet" "$CONFIGURE_LOG"; then +elif grep -q "Restored [1-9][0-9]* package(s) from NuGet" "$CONFIGURE_LOG"; then info "Confirmed vcpkg restored packages from the GitHub NuGet cache" else info "No NuGet restore line was printed. This usually means vcpkg packages were already installed in '$BUILD_DIR'." @@ -238,7 +250,8 @@ cmake --build "$BUILD_DIR" -- -j "$JOBS" if [[ "$RUN_TESTS" -eq 1 ]]; then info "Running tests" - ctest --test-dir "$BUILD_DIR" -j "$JOBS" --output-on-failure + ctest --test-dir "$BUILD_DIR" -j "$JOBS" --output-on-failure \ + -LE "(nonparallelizable_tests|long_running_tests|wasm_spec_tests)" fi info "Done" From 23ae29a2eb59561feafc2ebb10525625cdf4e28b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 21:16:02 +0000 Subject: [PATCH 05/14] Make cache build script skip tests by default --- BUILD.md | 11 +++++------ scripts/build-with-github-vcpkg-cache.sh | 9 +++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/BUILD.md b/BUILD.md index dc01fbcf8f..8ae7f90dac 100644 --- a/BUILD.md +++ b/BUILD.md @@ -154,8 +154,7 @@ scripts/build-with-github-vcpkg-cache.sh ``` The script bootstraps vcpkg, configures the GitHub Packages NuGet binary cache, -runs CMake with the repository vcpkg toolchain, builds the project, and runs -tests. +runs CMake with the repository vcpkg toolchain, and builds the project. When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` and stores cache files in `.ccache` by default. @@ -166,7 +165,7 @@ Useful options: scripts/build-with-github-vcpkg-cache.sh --build-dir build/release scripts/build-with-github-vcpkg-cache.sh --jobs 8 scripts/build-with-github-vcpkg-cache.sh --clean -scripts/build-with-github-vcpkg-cache.sh --skip-tests +scripts/build-with-github-vcpkg-cache.sh --run-tests ``` The script has three build modes: @@ -180,9 +179,9 @@ The script has three build modes: The default mode is `developer`. Developer mode keeps the local vcpkg caches between runs unless `--clean` is passed. GitHub Actions uses the same script -with `--skip-tests`; trusted pull requests run in `trusted-ci` mode, while fork -pull requests run in `forked-pr-ci` mode. CI runs tests in separate jobs after -archiving the build directory. +with trusted pull requests running in `trusted-ci` mode and fork pull requests +running in `forked-pr-ci` mode. CI runs tests in separate jobs after archiving +the build directory. ## Optional: Use the GitHub Packages vcpkg Binary Cache Manually diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index c3db7642be..074510f25f 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" JOBS="${JOBS:-$(nproc)}" -RUN_TESTS=1 +RUN_TESTS=0 CLEAN="${WIRE_SYSIO_CLEAN_BUILD:-}" NEEDS_CI_OWNERSHIP_FIX=0 BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" @@ -22,7 +22,8 @@ Options: --jobs N Parallel build jobs. Default: $JOBS --clean Remove vcpkg build artifacts before configuring. Default: enabled in CI modes, disabled in developer mode. - --skip-tests Configure and build only. + --run-tests Run the parallel-safe test subset after building. + --skip-tests Configure and build only. This is the default. --mode MODE Build mode: developer, trusted-ci, or forked-pr-ci. Default: $BUILD_MODE -h, --help Show this help. @@ -73,6 +74,10 @@ while [[ $# -gt 0 ]]; do CLEAN=1 shift ;; + --run-tests) + RUN_TESTS=1 + shift + ;; --skip-tests) RUN_TESTS=0 shift From c83f8d430bea695a0fa119ca48cf96802e2d3690 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 21:17:25 +0000 Subject: [PATCH 06/14] Remove redundant skip tests option --- .github/workflows/build_base.yaml | 3 +-- scripts/build-with-github-vcpkg-cache.sh | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index da8bc93330..cb8c7df2ad 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -96,8 +96,7 @@ jobs: scripts/build-with-github-vcpkg-cache.sh \ --mode "$BUILD_MODE" \ --build-dir build \ - --jobs "$BUILD_JOBS" \ - --skip-tests + --jobs "$BUILD_JOBS" echo "=== ccache statistics ===" ccache -s || true diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index 074510f25f..92ce250f32 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -23,7 +23,6 @@ Options: --clean Remove vcpkg build artifacts before configuring. Default: enabled in CI modes, disabled in developer mode. --run-tests Run the parallel-safe test subset after building. - --skip-tests Configure and build only. This is the default. --mode MODE Build mode: developer, trusted-ci, or forked-pr-ci. Default: $BUILD_MODE -h, --help Show this help. @@ -78,10 +77,6 @@ while [[ $# -gt 0 ]]; do RUN_TESTS=1 shift ;; - --skip-tests) - RUN_TESTS=0 - shift - ;; --mode) [[ $# -ge 2 ]] || fail "--mode requires a value." "Use '--mode developer', '--mode trusted-ci', or '--mode forked-pr-ci'." BUILD_MODE="$2" From 4f7deb7ce37bffc122facb5d6c76fd668f85146a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 19 May 2026 21:35:25 +0000 Subject: [PATCH 07/14] Clarify cache script test and fork behavior --- BUILD.md | 1 + scripts/build-with-github-vcpkg-cache.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/BUILD.md b/BUILD.md index 8ae7f90dac..5bb42d9e50 100644 --- a/BUILD.md +++ b/BUILD.md @@ -155,6 +155,7 @@ scripts/build-with-github-vcpkg-cache.sh The script bootstraps vcpkg, configures the GitHub Packages NuGet binary cache, runs CMake with the repository vcpkg toolchain, and builds the project. +It does not run tests unless `--run-tests` is passed. When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` and stores cache files in `.ccache` by default. diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index 92ce250f32..6cb5ba3146 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -139,6 +139,8 @@ info "Bootstrapping vcpkg" export VCPKG_DISABLE_METRICS="${VCPKG_DISABLE_METRICS:-1}" if [[ "$BUILD_MODE" == "forked-pr-ci" ]]; then + # Fork PRs cannot use package credentials, so they intentionally fall back to + # vcpkg's ephemeral runner-local binary cache and may rebuild dependencies. export VCPKG_BINARY_SOURCES="clear;default,readwrite" else if [[ "$BUILD_MODE" == "trusted-ci" ]]; then From c8016a31d8402722be4a0133c246b9aae24da01a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 20 May 2026 14:32:18 +0000 Subject: [PATCH 08/14] Use CMake presets for vcpkg cache builds --- .../vcpkg-triplets/x64-linux-release.cmake | 5 + .github/workflows/build_base.yaml | 7 +- .gitignore | 1 + BUILD.md | 30 ++- CMakePresets.json | 75 ++++++ scripts/build-with-github-vcpkg-cache.sh | 247 +++++++++++++++--- 6 files changed, 325 insertions(+), 40 deletions(-) create mode 100644 .github/vcpkg-triplets/x64-linux-release.cmake create mode 100644 CMakePresets.json diff --git a/.github/vcpkg-triplets/x64-linux-release.cmake b/.github/vcpkg-triplets/x64-linux-release.cmake new file mode 100644 index 0000000000..f5c4bd0090 --- /dev/null +++ b/.github/vcpkg-triplets/x64-linux-release.cmake @@ -0,0 +1,5 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +set(VCPKG_BUILD_TYPE release) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index cb8c7df2ad..fc2cd2b750 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -15,6 +15,10 @@ on: GH_TOKEN_DEV: required: true +# Trusted CI publishes vcpkg binary cache packages to GitHub Packages. Fork pull +# requests run with GitHub's reduced token permissions and the build script +# switches them to forked-pr-ci mode, which uses a local vcpkg cache and unsets +# package credentials before configuring. permissions: packages: write contents: read @@ -42,8 +46,7 @@ jobs: CCACHE_COMPRESS: "true" CCACHE_COMPRESSLEVEL: "6" VCPKG_DISABLE_METRICS: "1" - VCPKG_NUGET_FEED_URL: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json - VCPKG_NUGET_USER: ${{ github.repository_owner }} + VCPKG_NUGET_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json steps: - name: Allow safe directories run: git config --global --add safe.directory '*' diff --git a/.gitignore b/.gitignore index 73689241cb..9ffc97480e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ tmp \.#* CMakeCache.txt CMakeFiles +CMakeUserPresets.json cmake-build-*/ Makefile compile_commands.json diff --git a/BUILD.md b/BUILD.md index 5bb42d9e50..57c7025a3e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -146,16 +146,24 @@ The simplest way to build with the same vcpkg NuGet binary cache used by CI is to run: ```bash -export CC=/opt/clang/clang-18/bin/clang -export CXX=/opt/clang/clang-18/bin/clang++ -export CMAKE_PREFIX_PATH="/opt/clang/clang-18" - scripts/build-with-github-vcpkg-cache.sh ``` The script bootstraps vcpkg, configures the GitHub Packages NuGet binary cache, -runs CMake with the repository vcpkg toolchain, and builds the project. -It does not run tests unless `--run-tests` is passed. +runs CMake with a generated `dev-release` user preset, and +builds the project. The generated `CMakeUserPresets.json` captures the local +vcpkg binary cache environment for IDEs without storing package credentials. +The default presets use release-only vcpkg dependencies through the +`x64-linux-release` triplet, even for the +`dev-debug` preset. These presets are intended +for x86_64 Linux hosts; other architectures need a matching vcpkg triplet and +local preset override. + +The presets set `CMAKE_GENERATOR=Ninja` and select `clang-18` and `clang++-18` +by executable name, without hard-coding absolute paths. For a script run, set +`CC`, `CXX`, or `CMAKE_GENERATOR` when you need a specific toolchain. For +IDE-specific paths, create a local `CMakeUserPresets.json`. The script does not +run tests unless `--run-tests` is passed. When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` and stores cache files in `.ccache` by default. @@ -164,11 +172,17 @@ Useful options: ```bash scripts/build-with-github-vcpkg-cache.sh --build-dir build/release +scripts/build-with-github-vcpkg-cache.sh --preset dev-debug scripts/build-with-github-vcpkg-cache.sh --jobs 8 scripts/build-with-github-vcpkg-cache.sh --clean scripts/build-with-github-vcpkg-cache.sh --run-tests ``` +In developer mode, `--build-dir` is a build directory prefix: `dev-release` +uses `/release`, and `dev-debug` uses `/debug-release-deps`. In CI +modes, `--build-dir` is the exact CMake build directory so the workflow can +archive a stable `build` artifact. + The script has three build modes: - `developer`: local developer builds; reads packages from the GitHub Packages @@ -322,7 +336,9 @@ Choose **either** method A or B according to your preference. Method A (deb pack Wire Sysio also provides a Docker-based build environment for convenience. This is the easiest way to build and run Wire Sysio without manually installing all dependencies, as the Dockerfile encapsulates all requirements (including Clang 18, etc.) and build steps. -The provided Docker setup is intended for x86_64 hosts that can run Docker. The Dockerfile configures vcpkg with the `x64-linux` triplet. +The provided Docker setup is intended for x86_64 hosts that can run Docker. The +Dockerfile configures vcpkg with the `x64-linux` triplet, so arm64 hosts are not +supported by this Docker path unless they run the image under amd64 emulation. To build Wire Sysio using Docker: diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000000..b98895bf84 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,75 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "vcpkg-base", + "hidden": true, + "cacheVariables": { + "CMAKE_C_COMPILER": "clang-18", + "CMAKE_CXX_COMPILER": "clang++-18", + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/.github/vcpkg-triplets", + "VCPKG_TARGET_TRIPLET": "x64-linux-release", + "VCPKG_HOST_TRIPLET": "x64-linux-release", + "ENABLE_CCACHE": "ON", + "ENABLE_TESTS": "ON" + }, + "environment": { + "CMAKE_GENERATOR": "Ninja", + "VCPKG_DISABLE_METRICS": "1" + } + }, + { + "name": "vcpkg-github-release", + "displayName": "Release with GitHub vcpkg cache", + "description": "x86_64 Linux release build using release-only vcpkg dependencies; GitHub package cache is configured by the script or environment.", + "inherits": "vcpkg-base", + "binaryDir": "${sourceDir}/build/release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "vcpkg-github-debug-release-deps", + "displayName": "Debug with GitHub vcpkg cache release deps", + "description": "Debug build of Wire Sysio while keeping vcpkg dependencies release-only.", + "inherits": "vcpkg-github-release", + "binaryDir": "${sourceDir}/build/debug-release-deps", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "vcpkg-github-release-ci", + "displayName": "CI release with GitHub vcpkg cache", + "inherits": "vcpkg-github-release" + }, + { + "name": "vcpkg-local-release-fork", + "displayName": "Fork PR release with local vcpkg cache", + "inherits": "vcpkg-base", + "binaryDir": "${sourceDir}/build/fork-release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + } + ], + "buildPresets": [ + { + "name": "vcpkg-github-release", + "configurePreset": "vcpkg-github-release" + }, + { + "name": "vcpkg-github-debug-release-deps", + "configurePreset": "vcpkg-github-debug-release-deps" + }, + { + "name": "vcpkg-github-release-ci", + "configurePreset": "vcpkg-github-release-ci" + }, + { + "name": "vcpkg-local-release-fork", + "configurePreset": "vcpkg-local-release-fork" + } + ] +} diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index 6cb5ba3146..f135a39c2f 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -4,12 +4,24 @@ set -Eeuo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" +EFFECTIVE_BUILD_DIR="" JOBS="${JOBS:-$(nproc)}" RUN_TESTS=0 CLEAN="${WIRE_SYSIO_CLEAN_BUILD:-}" NEEDS_CI_OWNERSHIP_FIX=0 BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" -VCPKG_NUGET_FEED="${VCPKG_NUGET_FEED:-${VCPKG_NUGET_FEED_URL:-https://nuget.pkg.github.com/Wire-Network/index.json}}" +BUILD_MODE_EXPLICIT=0 +CMAKE_PRESET="${WIRE_SYSIO_CMAKE_PRESET:-}" +CMAKE_PRESET_EXPLICIT=0 +VCPKG_NUGET_FEED="${VCPKG_NUGET_FEED:-https://nuget.pkg.github.com/Wire-Network/index.json}" + +if [[ -n "${WIRE_SYSIO_BUILD_MODE:-}" ]]; then + BUILD_MODE_EXPLICIT=1 +fi + +if [[ -n "$CMAKE_PRESET" ]]; then + CMAKE_PRESET_EXPLICIT=1 +fi usage() { cat < "$file" <&1 | tee "$CONFIGURE_LOG" +cmake "${CMAKE_CONFIGURE_ARGS[@]}" 2>&1 | tee "$CONFIGURE_LOG" configure_status=${PIPESTATUS[0]} set -e @@ -239,20 +424,20 @@ if [[ "$configure_status" -ne 0 ]]; then fail "CMake configure failed." "Review $CONFIGURE_LOG. Common fixes:\n sudo apt-get install -y mono-complete ninja-build cmake\n gh auth refresh -h github.com -s read:packages\n rm -rf '$BUILD_DIR' and rerun this script after changing compilers." fi -if grep -q "Restored 0 package(s) from NuGet" "$CONFIGURE_LOG"; then +if grep -Eiq "Restored[[:space:]]+0([^0-9]|$).*package(s|\\(s\\))?.*from[[:space:]]+nuget" "$CONFIGURE_LOG"; then info "Warning: vcpkg reported zero NuGet restores. This can mean the ABI does not match CI, or that the packages were already installed in '$BUILD_DIR'." -elif grep -q "Restored [1-9][0-9]* package(s) from NuGet" "$CONFIGURE_LOG"; then +elif grep -Eiq "Restored[[:space:]]+[1-9][0-9]*([^0-9]|$).*package(s|\\(s\\))?.*from[[:space:]]+nuget" "$CONFIGURE_LOG"; then info "Confirmed vcpkg restored packages from the GitHub NuGet cache" else info "No NuGet restore line was printed. This usually means vcpkg packages were already installed in '$BUILD_DIR'." fi info "Building" -cmake --build "$BUILD_DIR" -- -j "$JOBS" +cmake --build "$EFFECTIVE_BUILD_DIR" -- -j "$JOBS" if [[ "$RUN_TESTS" -eq 1 ]]; then info "Running tests" - ctest --test-dir "$BUILD_DIR" -j "$JOBS" --output-on-failure \ + ctest --test-dir "$EFFECTIVE_BUILD_DIR" -j "$JOBS" --output-on-failure \ -LE "(nonparallelizable_tests|long_running_tests|wasm_spec_tests)" fi From 313d085ba8e838586a72fb03ac55b1186e8085ee Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 20 May 2026 16:08:34 +0000 Subject: [PATCH 09/14] Fix preset Ninja generator selection --- BUILD.md | 10 +++++----- CMakePresets.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/BUILD.md b/BUILD.md index 57c7025a3e..709c98f902 100644 --- a/BUILD.md +++ b/BUILD.md @@ -159,11 +159,11 @@ The default presets use release-only vcpkg dependencies through the for x86_64 Linux hosts; other architectures need a matching vcpkg triplet and local preset override. -The presets set `CMAKE_GENERATOR=Ninja` and select `clang-18` and `clang++-18` -by executable name, without hard-coding absolute paths. For a script run, set -`CC`, `CXX`, or `CMAKE_GENERATOR` when you need a specific toolchain. For -IDE-specific paths, create a local `CMakeUserPresets.json`. The script does not -run tests unless `--run-tests` is passed. +The presets use the `Ninja` generator and select `clang-18` and `clang++-18` by +executable name, without hard-coding absolute paths. For a script run, set +`CC` or `CXX` when you need a specific compiler. For IDE-specific paths or a +different generator, create a local `CMakeUserPresets.json`. The script does +not run tests unless `--run-tests` is passed. When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` and stores cache files in `.ccache` by default. diff --git a/CMakePresets.json b/CMakePresets.json index b98895bf84..5ebffe2149 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -4,6 +4,7 @@ { "name": "vcpkg-base", "hidden": true, + "generator": "Ninja", "cacheVariables": { "CMAKE_C_COMPILER": "clang-18", "CMAKE_CXX_COMPILER": "clang++-18", @@ -15,7 +16,6 @@ "ENABLE_TESTS": "ON" }, "environment": { - "CMAKE_GENERATOR": "Ninja", "VCPKG_DISABLE_METRICS": "1" } }, From 664b437212eb28731066997d37ddc085a3f94900 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 20 May 2026 17:44:07 +0000 Subject: [PATCH 10/14] Fix vcpkg NuGet cache publishing and local presets --- .github/workflows/build_base.yaml | 2 +- BUILD.md | 15 +++- scripts/build-with-github-vcpkg-cache.sh | 108 +++++++++++++++++++++-- 3 files changed, 114 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index fc2cd2b750..2c41d61b02 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -66,7 +66,7 @@ jobs: - name: Build id: build env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_DEV }} + GITHUB_TOKEN: ${{ github.token }} GITHUB_USER: ${{ github.repository_owner }} IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} run: | diff --git a/BUILD.md b/BUILD.md index 709c98f902..5c94eb20c1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -155,7 +155,7 @@ builds the project. The generated `CMakeUserPresets.json` captures the local vcpkg binary cache environment for IDEs without storing package credentials. The default presets use release-only vcpkg dependencies through the `x64-linux-release` triplet, even for the -`dev-debug` preset. These presets are intended +`dev-debug`, `dev-asan`, and `dev-ubsan` presets. These presets are intended for x86_64 Linux hosts; other architectures need a matching vcpkg triplet and local preset override. @@ -165,6 +165,9 @@ executable name, without hard-coding absolute paths. For a script run, set different generator, create a local `CMakeUserPresets.json`. The script does not run tests unless `--run-tests` is passed. +Developer user presets also export `CC` and `CXX` so vcpkg's compiler ABI +detection matches the script and CI when an IDE invokes CMake directly. + When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` and stores cache files in `.ccache` by default. @@ -173,15 +176,19 @@ Useful options: ```bash scripts/build-with-github-vcpkg-cache.sh --build-dir build/release scripts/build-with-github-vcpkg-cache.sh --preset dev-debug +scripts/build-with-github-vcpkg-cache.sh --preset dev-asan +scripts/build-with-github-vcpkg-cache.sh --preset dev-ubsan scripts/build-with-github-vcpkg-cache.sh --jobs 8 scripts/build-with-github-vcpkg-cache.sh --clean scripts/build-with-github-vcpkg-cache.sh --run-tests ``` In developer mode, `--build-dir` is a build directory prefix: `dev-release` -uses `/release`, and `dev-debug` uses `/debug-release-deps`. In CI -modes, `--build-dir` is the exact CMake build directory so the workflow can -archive a stable `build` artifact. +uses `/release`, `dev-debug` uses `/debug-release-deps`, `dev-asan` +uses `/asan`, and `dev-ubsan` uses `/ubsan`. The sanitizer presets +instrument Wire Sysio itself while reusing release-built vcpkg packages from +the GitHub NuGet cache. In CI modes, `--build-dir` is the exact CMake build +directory so the workflow can archive a stable `build` artifact. The script has three build modes: diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index f135a39c2f..108fceeecf 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -31,8 +31,9 @@ Build Wire Sysio with the GitHub Packages vcpkg NuGet binary cache. Options: --build-dir DIR Developer mode build directory prefix; dev-release - uses DIR/release and dev-debug uses - DIR/debug-release-deps. CI modes use DIR as the exact + uses DIR/release, dev-debug uses + DIR/debug-release-deps, dev-asan uses DIR/asan, and + dev-ubsan uses DIR/ubsan. CI modes use DIR as the exact CMake build directory. Default: $BUILD_DIR --preset NAME CMake configure/build preset. Defaults by mode: developer=dev-release, @@ -91,11 +92,50 @@ require_positive_integer() { fi } +shell_quote_paths() { + local path + printf '%q' "$1" + shift + for path in "$@"; do + printf ' %q' "$path" + done +} + +clean_build_artifacts() { + local paths=( + "$ROOT_DIR/vcpkg/buildtrees" + "$ROOT_DIR/vcpkg/packages" + "$ROOT_DIR/vcpkg/vcpkg_installed" + "$BUILD_DIR/vcpkg_installed" + "$HOME/.cache/vcpkg" + "$HOME/.vcpkg" + ) + local path + local root_owned_path + local blocked_paths=() + local quoted_paths + + for path in "${paths[@]}"; do + [[ -e "$path" || -L "$path" ]] || continue + root_owned_path="$(find "$path" ! -user "$(id -u)" -print -quit 2>/dev/null || true)" + if [[ -n "$root_owned_path" ]]; then + blocked_paths+=("$path") + fi + done + + if [[ "${#blocked_paths[@]}" -gt 0 ]]; then + quoted_paths="$(shell_quote_paths "${blocked_paths[@]}")" + fail "Cannot clean vcpkg build artifacts because some files are not owned by $(id -un)." "This usually happens after running the build in Docker or another root-owned environment. Repair ownership once, then rerun:\n sudo chown -R $(id -u):$(id -g) $quoted_paths\n scripts/build-with-github-vcpkg-cache.sh --clean" + fi + + rm -rf "${paths[@]}" +} + preset_mode() { local preset="$1" case "$preset" in - dev-release|dev-debug|vcpkg-github-release|vcpkg-github-debug-release-deps) + dev-release|dev-debug|dev-asan|dev-ubsan|vcpkg-github-release|vcpkg-github-debug-release-deps) printf 'developer' ;; vcpkg-github-release-ci) @@ -126,9 +166,13 @@ write_developer_user_presets() { local file="$ROOT_DIR/CMakeUserPresets.json" local marker='"wire-sysio/generated-by": "scripts/build-with-github-vcpkg-cache.sh"' local binary_sources_json + local asan_build_dir_json + local cc_json + local cxx_json local debug_build_dir_json local disable_metrics_json local release_build_dir_json + local ubsan_build_dir_json if [[ -f "$file" ]] && ! grep -q "$marker" "$file"; then info "Leaving existing CMakeUserPresets.json unchanged because it was not generated by this script" @@ -136,8 +180,12 @@ write_developer_user_presets() { fi binary_sources_json="$(printf '%s' "$VCPKG_BINARY_SOURCES" | jq -Rs .)" + cc_json="$(printf '%s' "$CC" | jq -Rs .)" + cxx_json="$(printf '%s' "$CXX" | jq -Rs .)" release_build_dir_json="$(developer_binary_dir "release" | jq -Rs .)" debug_build_dir_json="$(developer_binary_dir "debug-release-deps" | jq -Rs .)" + asan_build_dir_json="$(developer_binary_dir "asan" | jq -Rs .)" + ubsan_build_dir_json="$(developer_binary_dir "ubsan" | jq -Rs .)" disable_metrics_json="$(printf '%s' "$VCPKG_DISABLE_METRICS" | jq -Rs .)" cat > "$file" </dev/null 2>&1; then @@ -355,6 +442,9 @@ else fi fi +export CC="${CC:-clang-18}" +export CXX="${CXX:-clang++-18}" + if [[ "$BUILD_MODE" == "developer" ]]; then if ! write_developer_user_presets && [[ "$CMAKE_PRESET_EXPLICIT" -eq 0 ]]; then CMAKE_PRESET="vcpkg-github-release" @@ -367,6 +457,12 @@ if [[ "$BUILD_MODE" == "developer" ]]; then dev-debug|vcpkg-github-debug-release-deps) EFFECTIVE_BUILD_DIR="$BUILD_DIR/debug-release-deps" ;; + dev-asan) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/asan" + ;; + dev-ubsan) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/ubsan" + ;; *) EFFECTIVE_BUILD_DIR="$BUILD_DIR/release" ;; From 3e92e1a0517037da4702fa2c84414b1ae87475a3 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 20 May 2026 19:15:51 +0000 Subject: [PATCH 11/14] Debug vcpkg NuGet cache publishing --- .github/workflows/build_base.yaml | 7 ++++++- scripts/build-with-github-vcpkg-cache.sh | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 2c41d61b02..0bb7a86fc0 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -69,6 +69,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} GITHUB_USER: ${{ github.repository_owner }} IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + VCPKG_INSTALL_OPTIONS: --debug run: | echo "Building for ${{ matrix.platform }}" @@ -99,7 +100,11 @@ jobs: scripts/build-with-github-vcpkg-cache.sh \ --mode "$BUILD_MODE" \ --build-dir build \ - --jobs "$BUILD_JOBS" + --jobs "$BUILD_JOBS" \ + --configure-only + + echo "Diagnostic vcpkg run completed; stopping before project build and artifact upload." + exit 1 echo "=== ccache statistics ===" ccache -s || true diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh index 108fceeecf..1d97c6686b 100755 --- a/scripts/build-with-github-vcpkg-cache.sh +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -7,6 +7,7 @@ BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" EFFECTIVE_BUILD_DIR="" JOBS="${JOBS:-$(nproc)}" RUN_TESTS=0 +CONFIGURE_ONLY="${WIRE_SYSIO_CONFIGURE_ONLY:-0}" CLEAN="${WIRE_SYSIO_CLEAN_BUILD:-}" NEEDS_CI_OWNERSHIP_FIX=0 BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" @@ -43,6 +44,8 @@ Options: --clean Remove vcpkg build artifacts before configuring. Default: enabled in CI modes, disabled in developer mode. --run-tests Run the parallel-safe test subset after building. + --configure-only Stop after CMake configure. This completes the vcpkg + manifest install but skips the project build and tests. --mode MODE Build mode: developer, trusted-ci, or forked-pr-ci. Default: inferred from known --preset values, otherwise $BUILD_MODE. @@ -299,6 +302,10 @@ while [[ $# -gt 0 ]]; do RUN_TESTS=1 shift ;; + --configure-only) + CONFIGURE_ONLY=1 + shift + ;; --mode) [[ $# -ge 2 ]] || fail "--mode requires a value." "Use '--mode developer', '--mode trusted-ci', or '--mode forked-pr-ci'." BUILD_MODE="$2" @@ -499,6 +506,10 @@ if [[ -n "${CMAKE_PREFIX_PATH:-}" ]]; then CMAKE_CONFIGURE_ARGS+=(-DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH") fi +if [[ -n "${VCPKG_INSTALL_OPTIONS:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DVCPKG_INSTALL_OPTIONS="$VCPKG_INSTALL_OPTIONS") +fi + CONFIGURE_LOG="$EFFECTIVE_BUILD_DIR/cmake-configure.log" mkdir -p "$EFFECTIVE_BUILD_DIR" @@ -509,6 +520,9 @@ info "Build directory: $EFFECTIVE_BUILD_DIR" info "CMake preset: $CMAKE_PRESET" info "Build mode: $BUILD_MODE" info "vcpkg binary sources: $VCPKG_BINARY_SOURCES" +if [[ -n "${VCPKG_INSTALL_OPTIONS:-}" ]]; then + info "vcpkg install options: $VCPKG_INSTALL_OPTIONS" +fi info "Configuring CMake" set +e @@ -528,6 +542,11 @@ else info "No NuGet restore line was printed. This usually means vcpkg packages were already installed in '$BUILD_DIR'." fi +if [[ "$CONFIGURE_ONLY" == "1" ]]; then + info "Configure-only mode requested; stopping before project build" + exit 0 +fi + info "Building" cmake --build "$EFFECTIVE_BUILD_DIR" -- -j "$JOBS" From 734ebb52bd29558a0d103a7cbc25d711f0ae410a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 21 May 2026 19:23:24 +0000 Subject: [PATCH 12/14] Experiment with sysio-specific vcpkg triplet --- ...{x64-linux-release.cmake => x64-linux-sysio-release.cmake} | 0 BUILD.md | 2 +- CMakePresets.json | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename .github/vcpkg-triplets/{x64-linux-release.cmake => x64-linux-sysio-release.cmake} (100%) diff --git a/.github/vcpkg-triplets/x64-linux-release.cmake b/.github/vcpkg-triplets/x64-linux-sysio-release.cmake similarity index 100% rename from .github/vcpkg-triplets/x64-linux-release.cmake rename to .github/vcpkg-triplets/x64-linux-sysio-release.cmake diff --git a/BUILD.md b/BUILD.md index 5c94eb20c1..d1d66bad2a 100644 --- a/BUILD.md +++ b/BUILD.md @@ -154,7 +154,7 @@ runs CMake with a generated `dev-release` user preset, and builds the project. The generated `CMakeUserPresets.json` captures the local vcpkg binary cache environment for IDEs without storing package credentials. The default presets use release-only vcpkg dependencies through the -`x64-linux-release` triplet, even for the +`x64-linux-sysio-release` triplet, even for the `dev-debug`, `dev-asan`, and `dev-ubsan` presets. These presets are intended for x86_64 Linux hosts; other architectures need a matching vcpkg triplet and local preset override. diff --git a/CMakePresets.json b/CMakePresets.json index 5ebffe2149..fe75adaa0b 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -10,8 +10,8 @@ "CMAKE_CXX_COMPILER": "clang++-18", "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/.github/vcpkg-triplets", - "VCPKG_TARGET_TRIPLET": "x64-linux-release", - "VCPKG_HOST_TRIPLET": "x64-linux-release", + "VCPKG_TARGET_TRIPLET": "x64-linux-sysio-release", + "VCPKG_HOST_TRIPLET": "x64-linux-sysio-release", "ENABLE_CCACHE": "ON", "ENABLE_TESTS": "ON" }, From afb94d60555e001e08afb2b3c4b79b9879a24d4b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 21 May 2026 20:27:48 +0000 Subject: [PATCH 13/14] Clean up sysio triplet vcpkg packages --- .github/workflows/build_base.yaml | 87 +++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index 0bb7a86fc0..d9f50198a9 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -111,6 +111,93 @@ jobs: tar -pcz --exclude "*.o" build > build.tar.gz + - name: Delete sysio triplet vcpkg NuGet packages + if: always() + env: + GITHUB_TOKEN: ${{ github.token }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + + python3 - <<'PY' + import json + import os + import sys + import time + import urllib.error + import urllib.parse + import urllib.request + + owner = os.environ["OWNER"] + token = os.environ["GITHUB_TOKEN"] + suffix = "_x64-linux-sysio-release" + base = "https://api.github.com" + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "wire-sysio-vcpkg-cache-cleanup", + } + + def request(method, path): + req = urllib.request.Request(f"{base}{path}", headers=headers, method=method) + try: + with urllib.request.urlopen(req) as response: + body = response.read().decode("utf-8") + return response.status, json.loads(body) if body else None + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + if exc.code in (404, 409): + return exc.code, None + print(f"{method} {path} failed with HTTP {exc.code}: {body}", file=sys.stderr) + raise + + status, packages = request( + "GET", + f"/users/{urllib.parse.quote(owner)}/packages?package_type=nuget&per_page=100", + ) + if status != 200 or not packages: + print("No packages visible for cleanup") + sys.exit(0) + + targets = [package["name"] for package in packages if package["name"].endswith(suffix)] + if not targets: + print(f"No NuGet packages ending with {suffix} found") + sys.exit(0) + + print("Deleting package versions for:") + for target in targets: + print(f" {target}") + + for package_name in targets: + page = 1 + while True: + encoded = urllib.parse.quote(package_name, safe="") + status, versions = request( + "GET", + f"/users/{urllib.parse.quote(owner)}/packages/nuget/{encoded}/versions?per_page=100&page={page}", + ) + if status == 404 or not versions: + break + + for version in versions: + version_id = version["id"] + version_name = version.get("name", version_id) + delete_status, _ = request( + "DELETE", + f"/users/{urllib.parse.quote(owner)}/packages/nuget/{encoded}/versions/{version_id}", + ) + if delete_status == 204: + print(f"Deleted {package_name} {version_name}") + elif delete_status == 404: + print(f"Already deleted {package_name} {version_name}") + time.sleep(0.1) + + page += 1 + + print("Cleanup complete") + PY + - name: Show vcpkg logs if failure if: failure() run: | From 6789dcf5313c6538e825d863b5d1b39e3dda5197 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 21 May 2026 20:45:27 +0000 Subject: [PATCH 14/14] Paginate sysio triplet package cleanup --- .github/workflows/build_base.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index d9f50198a9..c13e4bd57b 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -152,11 +152,19 @@ jobs: print(f"{method} {path} failed with HTTP {exc.code}: {body}", file=sys.stderr) raise - status, packages = request( - "GET", - f"/users/{urllib.parse.quote(owner)}/packages?package_type=nuget&per_page=100", - ) - if status != 200 or not packages: + packages = [] + page = 1 + while True: + status, page_packages = request( + "GET", + f"/users/{urllib.parse.quote(owner)}/packages?package_type=nuget&per_page=100&page={page}", + ) + if status == 404 or not page_packages: + break + packages.extend(page_packages) + page += 1 + + if not packages: print("No packages visible for cleanup") sys.exit(0)