diff --git a/.cicd/platforms/asan.Dockerfile b/.cicd/platforms/asan.Dockerfile index e431ce247b..fa3145945f 100644 --- a/.cicd/platforms/asan.Dockerfile +++ b/.cicd/platforms/asan.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -75,4 +76,4 @@ COPY <<-EOF /extras.cmake EOF ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 -ENV ASAN_OPTIONS=detect_leaks=0 \ No newline at end of file +ENV ASAN_OPTIONS=detect_leaks=0 diff --git a/.cicd/platforms/asserton.Dockerfile b/.cicd/platforms/asserton.Dockerfile index f39066d1fa..8884d48442 100644 --- a/.cicd/platforms/asserton.Dockerfile +++ b/.cicd/platforms/asserton.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -78,4 +79,4 @@ COPY <<-EOF /extras.cmake set(SYSIO_ENABLE_RELEASE_BUILD_TEST "Off" CACHE BOOL "") EOF -ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 \ No newline at end of file +ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 diff --git a/.cicd/platforms/gcc.Dockerfile b/.cicd/platforms/gcc.Dockerfile index e2ffc30fbb..3f260f0d8f 100644 --- a/.cicd/platforms/gcc.Dockerfile +++ b/.cicd/platforms/gcc.Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ sudo \ golang \ python3-dev \ @@ -62,4 +63,4 @@ COPY <<-EOF /extras.cmake set(CMAKE_CXX_FLAGS "-Wall -Wextra -fdiagnostics-color=always" CACHE STRING "") EOF -ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 \ No newline at end of file +ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 diff --git a/.cicd/platforms/ubsan.Dockerfile b/.cicd/platforms/ubsan.Dockerfile index b69c82e863..8738b1a850 100644 --- a/.cicd/platforms/ubsan.Dockerfile +++ b/.cicd/platforms/ubsan.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ vim \ sudo \ doxygen \ @@ -79,4 +80,4 @@ COPY <<-EOF /extras.cmake EOF ENV SYSIO_PLATFORM_HAS_EXTRAS_CMAKE=1 -ENV UBSAN_OPTIONS=print_stacktrace=1,suppressions=/ubsan.supp \ No newline at end of file +ENV UBSAN_OPTIONS=print_stacktrace=1,suppressions=/ubsan.supp diff --git a/.cicd/platforms/ubuntu24.Dockerfile b/.cicd/platforms/ubuntu24.Dockerfile index 4504b80c0b..bef6769697 100644 --- a/.cicd/platforms/ubuntu24.Dockerfile +++ b/.cicd/platforms/ubuntu24.Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update && apt-get upgrade -y && \ zip \ unzip \ tar \ + mono-complete \ sudo \ golang \ python3-dev \ @@ -52,4 +53,4 @@ RUN apt-get update && apt-get upgrade -y && \ ENV CC=/usr/bin/clang-18 ENV CXX=/usr/bin/clang++-18 -ENV CMAKE_MAKE_PROGRAM=/usr/bin/ninja \ No newline at end of file +ENV CMAKE_MAKE_PROGRAM=/usr/bin/ninja diff --git a/.github/vcpkg-triplets/x64-linux-release.cmake b/.github/vcpkg-triplets/x64-linux-release.cmake new file mode 100644 index 0000000000..f5c4bd0090 --- /dev/null +++ b/.github/vcpkg-triplets/x64-linux-release.cmake @@ -0,0 +1,5 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) +set(VCPKG_BUILD_TYPE release) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d06959871c..39fb155218 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,9 @@ jobs: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml needs: [platform-cache] + permissions: + packages: write + contents: read secrets: GH_TOKEN_DEV: ${{ secrets.GH_TOKEN_DEV }} with: diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index c52e107aa5..0bb7a86fc0 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -15,8 +15,12 @@ on: GH_TOKEN_DEV: required: true +# Trusted CI publishes vcpkg binary cache packages to GitHub Packages. Fork pull +# requests run with GitHub's reduced token permissions and the build script +# switches them to forked-pr-ci mode, which uses a local vcpkg cache and unsets +# package credentials before configuring. permissions: - packages: read + packages: write contents: read defaults: @@ -41,6 +45,8 @@ jobs: CCACHE_MAXSIZE: "10G" CCACHE_COMPRESS: "true" CCACHE_COMPRESSLEVEL: "6" + VCPKG_DISABLE_METRICS: "1" + VCPKG_NUGET_FEED: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json steps: - name: Allow safe directories run: git config --global --add safe.directory '*' @@ -49,14 +55,6 @@ jobs: with: submodules: recursive - - name: Restore vcpkg binary cache - uses: actions/cache@v5 - with: - path: ${{ github.workspace }}/vcpkg-binary-cache - key: vcpkg-binaries-${{ matrix.platform }}-${{ hashFiles('vcpkg.json') }} - restore-keys: | - vcpkg-binaries-${{ matrix.platform }}- - - name: Restore ccache uses: actions/cache@v5 with: @@ -67,45 +65,47 @@ jobs: - name: Build id: build + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_USER: ${{ github.repository_owner }} + IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + VCPKG_INSTALL_OPTIONS: --debug run: | echo "Building for ${{ matrix.platform }}" # Use $GITHUB_WORKSPACE (container path /__w/...) not ${{ github.workspace }} - # (host path /home/runner/work/...) so ccache/vcpkg write to the mounted - # volume and persist for actions/cache to save. + # (host path /home/runner/work/...) so ccache writes to the mounted + # volume and persists for actions/cache to save. export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" - export VCPKG_BINARY_SOURCES="clear;files,$GITHUB_WORKSPACE/vcpkg-binary-cache,readwrite" - - # Clean intermediate vcpkg artifacts but preserve binary cache and downloads - rm -rf vcpkg/buildtrees vcpkg/packages vcpkg/vcpkg_installed \ - build/vcpkg_installed ~/.cache/vcpkg ~/.vcpkg - - mkdir -p "$GITHUB_WORKSPACE/vcpkg-binary-cache" - - ./vcpkg/bootstrap-vcpkg.sh chown -R $(id -u):$(id -g) $PWD # Reset ccache stats for this build ccache -z || true - cmake -B build -S . -G Ninja ${SYSIO_PLATFORM_HAS_EXTRAS_CMAKE:+-C /extras.cmake} \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_CXX_COMPILER=$CXX \ - -DCMAKE_MAKE_PROGRAM=$CMAKE_MAKE_PROGRAM \ - -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_CCACHE=ON \ - -DENABLE_TESTS=ON \ - ${CMAKE_PREFIX_PATH:+-DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH} - if [[ "${{ matrix.platform }}" == "gcc" ]]; then echo "Using reduced parallelism for gcc platform" - cmake --build build -- -j 8 + BUILD_JOBS=8 else - cmake --build build -- -j 11 + BUILD_JOBS=11 + fi + + BUILD_MODE=trusted-ci + if [[ "$IS_FORK_PR" == "true" ]]; then + echo "Using local vcpkg binary cache for fork pull request" + BUILD_MODE=forked-pr-ci + unset GITHUB_TOKEN GITHUB_USER fi + scripts/build-with-github-vcpkg-cache.sh \ + --mode "$BUILD_MODE" \ + --build-dir build \ + --jobs "$BUILD_JOBS" \ + --configure-only + + echo "Diagnostic vcpkg run completed; stopping before project build and artifact upload." + exit 1 + echo "=== ccache statistics ===" ccache -s || true diff --git a/.gitignore b/.gitignore index 73689241cb..9ffc97480e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ tmp \.#* CMakeCache.txt CMakeFiles +CMakeUserPresets.json cmake-build-*/ Makefile compile_commands.json diff --git a/BUILD.md b/BUILD.md index 99eb7510d1..5c94eb20c1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -135,57 +135,175 @@ This will build the vcpkg executable and set up the local vcpkg infrastructure. You are now ready to build Wire Sysio. -## Step 2 - Build +## Recommended: Build with the GitHub Packages vcpkg Cache -Make sure you are in the root of the `wire-sysio` repo, then perform the build with CMake and your chosen compiler. +Make sure you are in the root of the `wire-sysio` repo, then perform the build with the shared script used by CI and local developer builds. > ⚠️ **Memory/Parallelism Warning** ⚠️ > Building Wire Sysio from source can be resource-intensive. Some source files require **up to 4 GB of RAM** each to compile. If you use all CPU cores for parallel compilation (e.g. `make -j$(nproc)` or Ninja default parallelism), you may exhaust memory and encounter compiler crashes. Consider using a lower parallel job count (`-j`) if you run into memory issues or if you need to use your machine for other tasks during the build. -### Build Instructions +The simplest way to build with the same vcpkg NuGet binary cache used by CI is +to run: -First, ensure the environment is set to use **Clang 18** as the compiler: +```bash +scripts/build-with-github-vcpkg-cache.sh +``` + +The script bootstraps vcpkg, configures the GitHub Packages NuGet binary cache, +runs CMake with a generated `dev-release` user preset, and +builds the project. The generated `CMakeUserPresets.json` captures the local +vcpkg binary cache environment for IDEs without storing package credentials. +The default presets use release-only vcpkg dependencies through the +`x64-linux-release` triplet, even for the +`dev-debug`, `dev-asan`, and `dev-ubsan` presets. These presets are intended +for x86_64 Linux hosts; other architectures need a matching vcpkg triplet and +local preset override. + +The presets use the `Ninja` generator and select `clang-18` and `clang++-18` by +executable name, without hard-coding absolute paths. For a script run, set +`CC` or `CXX` when you need a specific compiler. For IDE-specific paths or a +different generator, create a local `CMakeUserPresets.json`. The script does +not run tests unless `--run-tests` is passed. + +Developer user presets also export `CC` and `CXX` so vcpkg's compiler ABI +detection matches the script and CI when an IDE invokes CMake directly. + +When `ccache` is installed, the CMake build uses it through `ENABLE_CCACHE=ON` +and stores cache files in `.ccache` by default. + +Useful options: ```bash -# Example: set CC and CXX to Clang 18 compilers (adjust path if installed elsewhere) -export CC=/opt/clang/clang-18/bin/clang -export CXX=/opt/clang/clang-18/bin/clang++ +scripts/build-with-github-vcpkg-cache.sh --build-dir build/release +scripts/build-with-github-vcpkg-cache.sh --preset dev-debug +scripts/build-with-github-vcpkg-cache.sh --preset dev-asan +scripts/build-with-github-vcpkg-cache.sh --preset dev-ubsan +scripts/build-with-github-vcpkg-cache.sh --jobs 8 +scripts/build-with-github-vcpkg-cache.sh --clean +scripts/build-with-github-vcpkg-cache.sh --run-tests +``` + +In developer mode, `--build-dir` is a build directory prefix: `dev-release` +uses `/release`, `dev-debug` uses `/debug-release-deps`, `dev-asan` +uses `/asan`, and `dev-ubsan` uses `/ubsan`. The sanitizer presets +instrument Wire Sysio itself while reusing release-built vcpkg packages from +the GitHub NuGet cache. In CI modes, `--build-dir` is the exact CMake build +directory so the workflow can archive a stable `build` artifact. + +The script has three build modes: + +- `developer`: local developer builds; reads packages from the GitHub Packages + NuGet cache and never publishes packages +- `trusted-ci`: trusted GitHub Actions runs; reads and writes the GitHub + Packages NuGet cache +- `forked-pr-ci`: fork pull-request runs; uses vcpkg's default local cache so + the workflow does not need package credentials + +The default mode is `developer`. Developer mode keeps the local vcpkg caches +between runs unless `--clean` is passed. GitHub Actions uses the same script +with trusted pull requests running in `trusted-ci` mode and fork pull requests +running in `forked-pr-ci` mode. CI runs tests in separate jobs after archiving +the build directory. + +## Optional: Use the GitHub Packages vcpkg Binary Cache Manually + +The project can restore vcpkg-built dependencies from the same NuGet-backed +binary cache used by CI. This is optional, but it avoids rebuilding large vcpkg +dependencies locally. + +To use the cache, you need: + +- a GitHub token that can read Wire-Network GitHub Packages +- `read:packages` scope on that token +- Mono, because vcpkg runs `nuget.exe` on Linux + +Install Mono: + +```bash +sudo apt-get install -y mono-complete +``` + +If you use the GitHub CLI, refresh the local token with package-read scope: + +```bash +gh auth refresh -h github.com -s read:packages +gh auth status +``` + +`gh auth status` should list `read:packages` in the token scopes. + +Configure the NuGet source: + +```bash +export GITHUB_TOKEN="$(gh auth token)" +export GITHUB_USER="$(gh api user --jq .login)" +export VCPKG_NUGET_FEED="https://nuget.pkg.github.com/Wire-Network/index.json" + +NUGET_EXE="$(./vcpkg/vcpkg fetch nuget | tail -n 1)" + +mono "$NUGET_EXE" sources remove -Name "github" >/dev/null 2>&1 || true +mono "$NUGET_EXE" sources add \ + -Name "github" \ + -Source "$VCPKG_NUGET_FEED" \ + -UserName "$GITHUB_USER" \ + -Password "$GITHUB_TOKEN" \ + -StorePasswordInClearText + +mono "$NUGET_EXE" setapikey "$GITHUB_TOKEN" -Source "$VCPKG_NUGET_FEED" ``` -Now run CMake to configure the build and generate build files. We recommend using **Ninja** as the build generator for faster builds (we installed `ninja-build` earlier): +Enable read-only binary cache restores for the current shell: ```bash +export VCPKG_FEATURE_FLAGS="manifests,binarycaching" +export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,read" +``` + +A successful restore looks like: + +```text +Restored 9 package(s) from NuGet +``` + +If vcpkg prints `Restored 0 package(s) from NuGet`, check: + +- `gh auth status` includes `read:packages` +- the compiler and platform match the CI platform image +- the vcpkg manifest, registry configuration, and dependency features match CI + +## Configure + +From the repository root: + +```bash +export CC=/opt/clang/clang-18/bin/clang +export CXX=/opt/clang/clang-18/bin/clang++ +export CMAKE_MAKE_PROGRAM=/usr/bin/ninja +export CMAKE_PREFIX_PATH="/opt/clang/clang-18" + cmake -B build -S . -G Ninja \ -DCMAKE_C_COMPILER="$CC" \ -DCMAKE_CXX_COMPILER="$CXX" \ + -DCMAKE_MAKE_PROGRAM="$CMAKE_MAKE_PROGRAM" \ -DCMAKE_TOOLCHAIN_FILE="$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="/opt/clang/clang-18" \ - -DENABLE_CCACHE=ON + -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ + -DENABLE_CCACHE=ON \ + -DENABLE_TESTS=ON ``` -In the above command: -- `-B build -S .` tells CMake to create (or use) the directory `build/` for the build files. -- `-G Ninja` chooses the Ninja generator. If you prefer Makefiles, you can omit this (default is Unix Makefiles), but ensure you adjust build commands accordingly. -- `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` are pointed to Clang 18 (from Step 1b). -- `CMAKE_TOOLCHAIN_FILE` points to the vcpkg toolchain file, so CMake will integrate vcpkg dependencies automatically. -- `CMAKE_BUILD_TYPE=Release` produces an optimized build. (You can use `Debug` for development, etc.) -- `CMAKE_PREFIX_PATH` is set to install location where you plan to install the CDT tooling (if not the system prefix `/usr`). -- `ENABLE_CCACHE=ON` will use **ccache** to cache compilation results (if `ccache` is installed). This can significantly speed up rebuilds. You can omit this or set `OFF` if you do not have ccache or do not want to use it. - -If CMake configuration is successful, it will generate the build files in the `build/` directory. Now proceed to compile: +## Build ```bash -cmake --build build -- -j$(nproc) +cmake --build build -- -j "$(nproc)" ``` -This will start the build process (using all available CPU cores by default). If you find your system running low on memory or becoming unresponsive, cancel the build (`Ctrl+C`) and re-run with a lower parallel job count, for example: +If you find your system running low on memory or becoming unresponsive, cancel +the build (`Ctrl+C`) and re-run with a lower parallel job count, for example: ```bash -cmake --build build -- -j4 -``` - -(to limit to 4 threads). Ninja will respect the `-j` flag similarly, or you can set the environment variable `CMAKE_BUILD_PARALLEL_LEVEL` before running the `cmake --build` command. +cmake --build build -- -j 4 +``` Once the build completes successfully, the Wire Sysio binaries and libraries will be available in `build/bin/` (and other subdirectories under `build/`). @@ -225,7 +343,9 @@ Choose **either** method A or B according to your preference. Method A (deb pack Wire Sysio also provides a Docker-based build environment for convenience. This is the easiest way to build and run Wire Sysio without manually installing all dependencies, as the Dockerfile encapsulates all requirements (including Clang 18, etc.) and build steps. -The provided Docker setup supports building on any modern 64-bit Linux or Apple Silicon (arm64) host that can run Docker (x86_64, arm64/aarch64). The container will produce Wire Sysio binaries for the same architecture as the host. +The provided Docker setup is intended for x86_64 hosts that can run Docker. The +Dockerfile configures vcpkg with the `x64-linux` triplet, so arm64 hosts are not +supported by this Docker path unless they run the image under amd64 emulation. To build Wire Sysio using Docker: @@ -234,24 +354,33 @@ To build Wire Sysio using Docker: ./scripts/docker-build.sh ``` -This script will build the Docker image (using the `./etc/docker/Dockerfile`) and compile Wire Sysio inside it. The final Wire Sysio build artifacts will be available within the Docker image (and can be copied out or used via Docker container). +This script will build the Docker image using `etc/docker/Dockerfile` and compile Wire Sysio inside it. The final Wire Sysio build artifacts are available inside the image under `/wire/wire-sysio/build/Release`. -By default, it builds the `app-build-repo` target (fetching the source from the repository). You can also build from your local source or choose other stages: +By default, it builds the `app-build-local` target using your current checkout as the source context. You can also choose other stages: -- To build using **local source context** (your cloned repository code), use the `app-build-local` target: +- To build using **local source context** explicitly, use the `app-build-local` target: ```bash ./scripts/docker-build.sh --target=app-build-local ``` -- To specify a different Git branch or repository for the source, set the `SYSIO_BRANCH` or `REPO` build args via the script (see script help for usage). +- To build from the remote repository source, use the `app-build-repo` target: + ```bash + ./scripts/docker-build.sh --target=app-build-repo --sysio-branch=master + ``` +- To build the repository source with Wire CDT as well, use the `cdt-build-repo` target: + ```bash + ./scripts/docker-build.sh --target=cdt-build-repo --sysio-branch=master --cdt-branch=master + ``` -- To tag the resulting Docker image with a specific name (default tag is `wire/sysio:latest`), use the `--tag` option: +- To tag the resulting Docker image with a specific name (default tag is `wire/sysio`), use the `--tag` option: ```bash ./scripts/docker-build.sh --target=app-build-local --tag=wire/sysio:mybuild ``` -- There is an additional flag that can be used to specify which Ubuntu version your image builds on (defaults Ubuntu 24.04) +- To specify which Ubuntu base image tag to build on, use `--ubuntu-tag` (defaults to `24.04`): ```bash ./scripts/docker-build.sh --ubuntu-tag=24.04 ``` + +The script requires at least 32 GiB of available memory and passes `--memory 32G` to `docker build`. After the script finishes, you will have a Docker image with Wire Sysio built inside. You can run this image or extract the built binaries. (Refer to the Dockerfile targets and the script for details on where the binaries reside in each stage.) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000000..5ebffe2149 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,75 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "vcpkg-base", + "hidden": true, + "generator": "Ninja", + "cacheVariables": { + "CMAKE_C_COMPILER": "clang-18", + "CMAKE_CXX_COMPILER": "clang++-18", + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/.github/vcpkg-triplets", + "VCPKG_TARGET_TRIPLET": "x64-linux-release", + "VCPKG_HOST_TRIPLET": "x64-linux-release", + "ENABLE_CCACHE": "ON", + "ENABLE_TESTS": "ON" + }, + "environment": { + "VCPKG_DISABLE_METRICS": "1" + } + }, + { + "name": "vcpkg-github-release", + "displayName": "Release with GitHub vcpkg cache", + "description": "x86_64 Linux release build using release-only vcpkg dependencies; GitHub package cache is configured by the script or environment.", + "inherits": "vcpkg-base", + "binaryDir": "${sourceDir}/build/release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "vcpkg-github-debug-release-deps", + "displayName": "Debug with GitHub vcpkg cache release deps", + "description": "Debug build of Wire Sysio while keeping vcpkg dependencies release-only.", + "inherits": "vcpkg-github-release", + "binaryDir": "${sourceDir}/build/debug-release-deps", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "vcpkg-github-release-ci", + "displayName": "CI release with GitHub vcpkg cache", + "inherits": "vcpkg-github-release" + }, + { + "name": "vcpkg-local-release-fork", + "displayName": "Fork PR release with local vcpkg cache", + "inherits": "vcpkg-base", + "binaryDir": "${sourceDir}/build/fork-release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + } + ], + "buildPresets": [ + { + "name": "vcpkg-github-release", + "configurePreset": "vcpkg-github-release" + }, + { + "name": "vcpkg-github-debug-release-deps", + "configurePreset": "vcpkg-github-debug-release-deps" + }, + { + "name": "vcpkg-github-release-ci", + "configurePreset": "vcpkg-github-release-ci" + }, + { + "name": "vcpkg-local-release-fork", + "configurePreset": "vcpkg-local-release-fork" + } + ] +} diff --git a/scripts/build-with-github-vcpkg-cache.sh b/scripts/build-with-github-vcpkg-cache.sh new file mode 100755 index 0000000000..1d97c6686b --- /dev/null +++ b/scripts/build-with-github-vcpkg-cache.sh @@ -0,0 +1,559 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build}" +EFFECTIVE_BUILD_DIR="" +JOBS="${JOBS:-$(nproc)}" +RUN_TESTS=0 +CONFIGURE_ONLY="${WIRE_SYSIO_CONFIGURE_ONLY:-0}" +CLEAN="${WIRE_SYSIO_CLEAN_BUILD:-}" +NEEDS_CI_OWNERSHIP_FIX=0 +BUILD_MODE="${WIRE_SYSIO_BUILD_MODE:-developer}" +BUILD_MODE_EXPLICIT=0 +CMAKE_PRESET="${WIRE_SYSIO_CMAKE_PRESET:-}" +CMAKE_PRESET_EXPLICIT=0 +VCPKG_NUGET_FEED="${VCPKG_NUGET_FEED:-https://nuget.pkg.github.com/Wire-Network/index.json}" + +if [[ -n "${WIRE_SYSIO_BUILD_MODE:-}" ]]; then + BUILD_MODE_EXPLICIT=1 +fi + +if [[ -n "$CMAKE_PRESET" ]]; then + CMAKE_PRESET_EXPLICIT=1 +fi + +usage() { + cat <&2 + exit 1 +} + +info() { + printf '==> %s\n' "$*" +} + +require_command() { + local command_name="$1" + local package_hint="$2" + + if ! command -v "$command_name" >/dev/null 2>&1; then + fail "'$command_name' is not installed." "Install it with:\n sudo apt-get install -y $package_hint" + fi +} + +require_positive_integer() { + local value="$1" + local name="$2" + + if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then + fail "$name must be a positive integer." "Use '$0 --jobs $(nproc)' or set JOBS to a positive integer." + fi +} + +shell_quote_paths() { + local path + printf '%q' "$1" + shift + for path in "$@"; do + printf ' %q' "$path" + done +} + +clean_build_artifacts() { + local paths=( + "$ROOT_DIR/vcpkg/buildtrees" + "$ROOT_DIR/vcpkg/packages" + "$ROOT_DIR/vcpkg/vcpkg_installed" + "$BUILD_DIR/vcpkg_installed" + "$HOME/.cache/vcpkg" + "$HOME/.vcpkg" + ) + local path + local root_owned_path + local blocked_paths=() + local quoted_paths + + for path in "${paths[@]}"; do + [[ -e "$path" || -L "$path" ]] || continue + root_owned_path="$(find "$path" ! -user "$(id -u)" -print -quit 2>/dev/null || true)" + if [[ -n "$root_owned_path" ]]; then + blocked_paths+=("$path") + fi + done + + if [[ "${#blocked_paths[@]}" -gt 0 ]]; then + quoted_paths="$(shell_quote_paths "${blocked_paths[@]}")" + fail "Cannot clean vcpkg build artifacts because some files are not owned by $(id -un)." "This usually happens after running the build in Docker or another root-owned environment. Repair ownership once, then rerun:\n sudo chown -R $(id -u):$(id -g) $quoted_paths\n scripts/build-with-github-vcpkg-cache.sh --clean" + fi + + rm -rf "${paths[@]}" +} + +preset_mode() { + local preset="$1" + + case "$preset" in + dev-release|dev-debug|dev-asan|dev-ubsan|vcpkg-github-release|vcpkg-github-debug-release-deps) + printf 'developer' + ;; + vcpkg-github-release-ci) + printf 'trusted-ci' + ;; + vcpkg-local-release-fork) + printf 'forked-pr-ci' + ;; + esac +} + +developer_binary_dir() { + local suffix="$1" + local absolute_dir + local relative_dir + + absolute_dir="$(realpath -m "$BUILD_DIR/$suffix")" + relative_dir="$(realpath -m --relative-to="$ROOT_DIR" "$absolute_dir")" + + if [[ "$relative_dir" != .. && "$relative_dir" != ../* ]]; then + printf '${sourceDir}/%s' "$relative_dir" + else + printf '%s' "$absolute_dir" + fi +} + +write_developer_user_presets() { + local file="$ROOT_DIR/CMakeUserPresets.json" + local marker='"wire-sysio/generated-by": "scripts/build-with-github-vcpkg-cache.sh"' + local binary_sources_json + local asan_build_dir_json + local cc_json + local cxx_json + local debug_build_dir_json + local disable_metrics_json + local release_build_dir_json + local ubsan_build_dir_json + + if [[ -f "$file" ]] && ! grep -q "$marker" "$file"; then + info "Leaving existing CMakeUserPresets.json unchanged because it was not generated by this script" + return 1 + fi + + binary_sources_json="$(printf '%s' "$VCPKG_BINARY_SOURCES" | jq -Rs .)" + cc_json="$(printf '%s' "$CC" | jq -Rs .)" + cxx_json="$(printf '%s' "$CXX" | jq -Rs .)" + release_build_dir_json="$(developer_binary_dir "release" | jq -Rs .)" + debug_build_dir_json="$(developer_binary_dir "debug-release-deps" | jq -Rs .)" + asan_build_dir_json="$(developer_binary_dir "asan" | jq -Rs .)" + ubsan_build_dir_json="$(developer_binary_dir "ubsan" | jq -Rs .)" + disable_metrics_json="$(printf '%s' "$VCPKG_DISABLE_METRICS" | jq -Rs .)" + + cat > "$file" </dev/null 2>&1; then + fail "GitHub CLI is not authenticated." "Authenticate and request package-read scope:\n gh auth login -h github.com\n gh auth refresh -h github.com -s read:packages" + fi + + GH_STATUS="$(gh auth status -h github.com 2>&1 || true)" + if [[ "$GH_STATUS" != *"read:packages"* ]]; then + fail "GitHub CLI token is missing the 'read:packages' scope." "Refresh the token scope, then rerun this script:\n gh auth refresh -h github.com -s read:packages\n gh auth status -h github.com" + fi + + GITHUB_TOKEN="${GITHUB_TOKEN:-$(gh auth token)}" + GITHUB_USER="${GITHUB_USER:-$(gh api user --jq .login)}" + + if [[ -z "$GITHUB_TOKEN" || -z "$GITHUB_USER" ]]; then + fail "Could not resolve GitHub token or username." "Set GITHUB_TOKEN and GITHUB_USER explicitly, or fix GitHub CLI authentication with 'gh auth login'." + fi + fi + + NUGET_EXE="$("$ROOT_DIR/vcpkg/vcpkg" fetch nuget | tail -n 1)" + if [[ ! -f "$NUGET_EXE" ]]; then + fail "vcpkg did not return a usable nuget.exe path." "Run '$ROOT_DIR/vcpkg/vcpkg fetch nuget' and fix any reported vcpkg download errors." + fi + + info "Configuring GitHub Packages NuGet source" + mono "$NUGET_EXE" sources remove -Name "github" >/dev/null 2>&1 || true + # NuGet on Linux under Mono needs this flag to persist the feed credential. + mono "$NUGET_EXE" sources add \ + -Name "github" \ + -Source "$VCPKG_NUGET_FEED" \ + -UserName "$GITHUB_USER" \ + -Password "$GITHUB_TOKEN" \ + -StorePasswordInClearText >/dev/null + mono "$NUGET_EXE" setapikey "$GITHUB_TOKEN" -Source "$VCPKG_NUGET_FEED" >/dev/null + + if [[ "$BUILD_MODE" == "developer" ]]; then + export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,read" + else + export VCPKG_BINARY_SOURCES="clear;nuget,$VCPKG_NUGET_FEED,readwrite" + fi +fi + +export CC="${CC:-clang-18}" +export CXX="${CXX:-clang++-18}" + +if [[ "$BUILD_MODE" == "developer" ]]; then + if ! write_developer_user_presets && [[ "$CMAKE_PRESET_EXPLICIT" -eq 0 ]]; then + CMAKE_PRESET="vcpkg-github-release" + info "Using committed preset '$CMAKE_PRESET' for this run" + fi +fi + +if [[ "$BUILD_MODE" == "developer" ]]; then + case "$CMAKE_PRESET" in + dev-debug|vcpkg-github-debug-release-deps) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/debug-release-deps" + ;; + dev-asan) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/asan" + ;; + dev-ubsan) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/ubsan" + ;; + *) + EFFECTIVE_BUILD_DIR="$BUILD_DIR/release" + ;; + esac +else + EFFECTIVE_BUILD_DIR="$BUILD_DIR" +fi + +export CCACHE_DIR="${CCACHE_DIR:-$ROOT_DIR/.ccache}" +export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-10G}" + +CMAKE_CONFIGURE_ARGS=( + --preset "$CMAKE_PRESET" + -B "$EFFECTIVE_BUILD_DIR" +) + +if [[ -n "${SYSIO_PLATFORM_HAS_EXTRAS_CMAKE:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-C /extras.cmake) +fi + +if [[ -n "${CC:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DCMAKE_C_COMPILER="$CC") +fi + +if [[ -n "${CXX:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DCMAKE_CXX_COMPILER="$CXX") +fi + +if [[ -n "${CMAKE_MAKE_PROGRAM:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DCMAKE_MAKE_PROGRAM="$CMAKE_MAKE_PROGRAM") +fi + +if [[ -n "${CMAKE_PREFIX_PATH:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH") +fi + +if [[ -n "${VCPKG_INSTALL_OPTIONS:-}" ]]; then + CMAKE_CONFIGURE_ARGS+=(-DVCPKG_INSTALL_OPTIONS="$VCPKG_INSTALL_OPTIONS") +fi + +CONFIGURE_LOG="$EFFECTIVE_BUILD_DIR/cmake-configure.log" +mkdir -p "$EFFECTIVE_BUILD_DIR" + +if [[ "$BUILD_MODE" == "developer" ]]; then + info "Build directory prefix: $BUILD_DIR" +fi +info "Build directory: $EFFECTIVE_BUILD_DIR" +info "CMake preset: $CMAKE_PRESET" +info "Build mode: $BUILD_MODE" +info "vcpkg binary sources: $VCPKG_BINARY_SOURCES" +if [[ -n "${VCPKG_INSTALL_OPTIONS:-}" ]]; then + info "vcpkg install options: $VCPKG_INSTALL_OPTIONS" +fi + +info "Configuring CMake" +set +e +cmake "${CMAKE_CONFIGURE_ARGS[@]}" 2>&1 | tee "$CONFIGURE_LOG" +configure_status=${PIPESTATUS[0]} +set -e + +if [[ "$configure_status" -ne 0 ]]; then + fail "CMake configure failed." "Review $CONFIGURE_LOG. Common fixes:\n sudo apt-get install -y mono-complete ninja-build cmake\n gh auth refresh -h github.com -s read:packages\n rm -rf '$BUILD_DIR' and rerun this script after changing compilers." +fi + +if grep -Eiq "Restored[[:space:]]+0([^0-9]|$).*package(s|\\(s\\))?.*from[[:space:]]+nuget" "$CONFIGURE_LOG"; then + info "Warning: vcpkg reported zero NuGet restores. This can mean the ABI does not match CI, or that the packages were already installed in '$BUILD_DIR'." +elif grep -Eiq "Restored[[:space:]]+[1-9][0-9]*([^0-9]|$).*package(s|\\(s\\))?.*from[[:space:]]+nuget" "$CONFIGURE_LOG"; then + info "Confirmed vcpkg restored packages from the GitHub NuGet cache" +else + info "No NuGet restore line was printed. This usually means vcpkg packages were already installed in '$BUILD_DIR'." +fi + +if [[ "$CONFIGURE_ONLY" == "1" ]]; then + info "Configure-only mode requested; stopping before project build" + exit 0 +fi + +info "Building" +cmake --build "$EFFECTIVE_BUILD_DIR" -- -j "$JOBS" + +if [[ "$RUN_TESTS" -eq 1 ]]; then + info "Running tests" + ctest --test-dir "$EFFECTIVE_BUILD_DIR" -j "$JOBS" --output-on-failure \ + -LE "(nonparallelizable_tests|long_running_tests|wasm_spec_tests)" +fi + +info "Done"