diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f3609230636..8b29954a77d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,108 +1,10 @@ -FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/cpp:1-ubuntu-22.04 +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 -# Install software-properties-common for add-apt-repository -RUN apt-get -y update && apt-get -y install software-properties-common && add-apt-repository ppa:ubuntu-toolchain-r/test +# CPM supplies project libraries at configure time. The image contains only +# platform prerequisites and developer tools. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates ccache clang cmake gdb git libidn11-dev libssl-dev lld ninja-build \ + pkg-config python3 ragel yasm \ + && rm -rf /var/lib/apt/lists/* -# Install C++ tools and libraries -RUN apt-get -y update && apt-get -y install \ - git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Install CMake -ENV CMAKE_VERSION=3.27.7 -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ - -q -O cmake-install.sh \ - && chmod u+x cmake-install.sh \ - && ./cmake-install.sh --skip-license --prefix=/usr/local \ - && rm cmake-install.sh - -# Install LLVM -ENV LLVM_VERSION=16 -RUN wget https://apt.llvm.org/llvm.sh && \ - chmod u+x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} - -# Update alternatives to use clang-16 by default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000 - -# Update alternatives to use gcc-13 by default -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - -# Install abseil-cpp -ENV ABSEIL_CPP_VERSION=20230802.0 -ENV ABSEIL_CPP_INSTALL_DIR=~/ydb_deps/absl -RUN wget -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \ - tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} - -# Install protobuf -ENV PROTOBUF_VERSION=25.0 -ENV PROTOBUF_INSTALL_DIR=~/ydb_deps/protobuf -RUN wget -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ - tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} - -# Install grpc -ENV GRPC_VERSION=1.60.2 -ENV GRPC_INSTALL_DIR=~/ydb_deps/grpc -RUN wget -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ - tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ - -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR} - -# Install base64 -ENV BASE64_VERSION=0.5.2 -RUN wget -O base64-${BASE64_VERSION}.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v${BASE64_VERSION}.tar.gz && \ - tar -xvzf base64-${BASE64_VERSION}.tar.gz && cd base64-${BASE64_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release --target install - -# Install brotli -ENV BROTLI_VERSION=1.1.0 -RUN wget -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ - tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release --target install - -# Install jwt-cpp -ENV JWT_CPP_VERSION=0.7.0 -RUN wget -O jwt-cpp-${JWT_CPP_VERSION}.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_CPP_VERSION}.tar.gz && \ - tar -xvzf jwt-cpp-${JWT_CPP_VERSION}.tar.gz && cd jwt-cpp-${JWT_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release --target install - -# Install ccache 4.8.1 or above -ENV CCACHE_VERSION=4.8.1 -RUN wget https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && tar -xf ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && cp ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/local/bin/ \ - && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64 ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz - -# Install ydb cli 2.16.0 -ENV YDB_VERSION=2.16.0 -RUN wget https://storage.yandexcloud.net/yandexcloud-ydb/release/${YDB_VERSION}/linux/amd64/ydb && \ - chmod u+x ydb && \ - mv ydb /usr/local/bin/ +ENV CPM_SOURCE_CACHE=/workspaces/.cache/cpm diff --git a/.devcontainer/configure.sh b/.devcontainer/configure.sh index 2fb189822ec..114428a9167 100755 --- a/.devcontainer/configure.sh +++ b/.devcontainer/configure.sh @@ -1,8 +1,6 @@ #!/bin/sh -mkdir -p build -git submodule update --init --recursive -ccache -o cache_dir=/root/.ccache +ccache --set-config=cache_dir=/root/.ccache cmake --preset release-test-clang if which ydb > /dev/null 2>&1; then diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 067965ebee1..82e8d69f97c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,7 @@ 8765 ], // Use 'initializeCommand' to run commands before the container is created. - "initializeCommand": "mkdir -p ${localEnv:HOME}/.ccache && cd \"${localWorkspaceFolder}\" && git config --local user.email \"$(git config user.email)\" && git config --local user.name \"$(git config user.name)\"", + "initializeCommand": "mkdir -p ${localEnv:HOME}/.ccache ${localEnv:HOME}/.cache/ydb-cpp-sdk-cpm && cd \"${localWorkspaceFolder}\" && git config --local user.email \"$(git config user.email)\" && git config --local user.name \"$(git config user.name)\"", // Use 'postStartCommand' to run commands after the container is started. "postStartCommand": ".devcontainer/configure.sh", // Configure tool-specific properties. @@ -40,7 +40,8 @@ } }, "mounts": [ - "source=${localEnv:HOME}/.ccache,target=/root/.ccache,type=bind,consistency=cached" + "source=${localEnv:HOME}/.ccache,target=/root/.ccache,type=bind,consistency=cached", + "source=${localEnv:HOME}/.cache/ydb-cpp-sdk-cpm,target=/workspaces/.cache/cpm,type=bind,consistency=cached" ], // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. "remoteUser": "root" diff --git a/.github/actions/gcovr/run_gcovr.sh b/.github/actions/gcovr/run_gcovr.sh index f2449b6e56b..407fd36a56d 100755 --- a/.github/actions/gcovr/run_gcovr.sh +++ b/.github/actions/gcovr/run_gcovr.sh @@ -12,6 +12,7 @@ gcovr_args=( --root "${SOURCE_ROOT}" --object-directory "${BUILD_DIR}" --gcov-executable "${GCOV_EXECUTABLE}" + --gcov-ignore-parse-errors negative_hits.warn_once_per_file --print-summary --json-summary "${OUTPUT_DIR}/summary.json" --html --html-details @@ -21,7 +22,6 @@ gcovr_args=( --exclude '.*tests/.*' --merge-lines --exclude '.*/_deps/.*' - --merge-lines --filter 'src/' --filter 'include/ydb-cpp-sdk/' --filter 'plugins/' diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index 2e6f36d43ea..da8dacbc21b 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -1,94 +1,30 @@ name: Prepare VM for YDB SDK build -description: Install required packages +description: Install platform prerequisites; CPM supplies project libraries inputs: mode: - description: "Setup scope: build installs full build dependencies; runtime installs only test runtime dependencies." + description: "Setup scope: build installs tools and headers; runtime installs shared runtime libraries." required: false default: build runs: - using: "composite" + using: composite steps: - name: Install CMake and Ninja + if: ${{ inputs.mode == 'build' }} uses: lukka/get-cmake@v3.27.7 - - name: Install runtime dependencies - shell: bash - run: | - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get -y update - sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - - - name: Install build dependencies - if: ${{ inputs.mode == 'build' }} + - name: Install platform prerequisites shell: bash run: | - # Install ccache - (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ - sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) - - ccache -o base_dir=`realpath ..` - ccache -o cache_dir=~/.ccache - ccache -z - export ENABLE_CCACHE="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - - wget https://apt.llvm.org/llvm.sh - chmod u+x llvm.sh - sudo ./llvm.sh 18 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 10000 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 10000 - - # Install abseil-cpp - wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz - tar -xvzf abseil-cpp-20230802.0.tar.gz - cd abseil-cpp-20230802.0 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/absl - cd ../../ - - # Install protobuf - wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz - tar -xvzf protobuf-25.0.tar.gz - cd protobuf-25.0 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ - -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/protobuf - cd ../../ - - # Install gRPC - wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz - tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_PREFIX_PATH="${HOME}/ydb_deps/absl;${HOME}/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ - -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/grpc - cd ../../ - - # Install brotli - wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz - tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/brotli - cd ../../ - - # Clean up - ccache -s - sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-25.0.tar.gz grpc-1.60.2.tar.gz \ - brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \ - protobuf-25.0 grpc-1.60.2 brotli-1.1.0 + sudo apt-get -y update + if [[ '${{ inputs.mode }}' == 'build' ]]; then + sudo apt-get -y install \ + ccache clang g++ gcc git libidn11-dev libssl-dev lld ninja-build \ + pkg-config python3 ragel yasm + ccache --set-config=base_dir="$(realpath ..)" + ccache --set-config=cache_dir="$HOME/.ccache" + ccache --zero-stats + else + sudo apt-get -y install libidn12 libssl3 libstdc++6 + fi diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh index 28f5ca8c8ab..f6828c80354 100755 --- a/.github/scripts/copy_sources.sh +++ b/.github/scripts/copy_sources.sh @@ -204,7 +204,6 @@ sync_upstream_tree "$1" "$2" "$tmp_dir" tools/enum_parser managed sync_upstream_tree "$1" "$2" "$tmp_dir" tools/rescompiler managed cp $2/.gitignore $tmp_dir -cp $2/.gitmodules $tmp_dir cp $2/CMakePresets.json $tmp_dir cp $2/CMakeLists.txt $tmp_dir cp $2/LICENSE $tmp_dir diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c6d978e0df8..64b8c66cfb7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,7 +15,9 @@ concurrency: jobs: coverage: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm permissions: contents: read id-token: write @@ -38,29 +40,21 @@ jobs: uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' - with: - submodules: true - - - name: Prepare coverage cache key - id: coverage-cache-key - shell: bash - run: | - echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT" - - name: Restore coverage build cache - id: coverage-build-cache - uses: actions/cache/restore@v4 + - name: Restore CPM and compiler caches + uses: actions/cache@v4 with: - path: build - key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .cache/cpm + ~/.ccache + key: coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }} restore-keys: | - ${{ steps.coverage-cache-key.outputs.prefix }}- + coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}- - name: Install dependencies uses: ./.github/actions/prepare_vm @@ -95,7 +89,7 @@ jobs: with: build-directory: build source-root: ${{ github.workspace }} - gcov-executable: gcov-13 + gcov-executable: gcov - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 @@ -115,18 +109,3 @@ jobs: name: coverage-report path: build/coverage/ retention-days: 14 - - - name: Clean coverage data before cache save - if: github.event_name != 'pull_request' && success() - shell: bash - run: | - find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete - rm -rf build/coverage - cmake -E remove_directory build/util/system/testing_out_stuff - - - name: Save coverage build cache - if: github.event_name != 'pull_request' && success() && steps.coverage-build-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: build - key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index eed16ccbebb..0e34be68933 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -14,7 +14,9 @@ concurrency: jobs: main: name: Examples - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm strategy: fail-fast: false matrix: @@ -39,26 +41,27 @@ jobs: uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' + - name: Restore CPM sources + uses: actions/cache@v4 with: - submodules: true + path: .cache/cpm + key: cpm-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }} - name: Prepare ccache timestamp id: ccache_cache_timestamp - shell: cmake -P {0} + shell: bash run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") + echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: ~/.ccache - key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + key: ubuntu-24.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | - ubuntu-22.04-ccache-${{ matrix.compiler }}- + ubuntu-24.04-ccache-${{ matrix.compiler }}- - name: Install dependencies uses: ./.github/actions/prepare_vm - name: Build diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml index ba9e7dc7177..a260a1a1ecd 100644 --- a/.github/workflows/import.yaml +++ b/.github/workflows/import.yaml @@ -10,7 +10,6 @@ jobs: - name: Checkout ydb-cpp-sdk uses: actions/checkout@v4 with: - submodules: true path: ydb-cpp-sdk - name: Checkout ydb diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 9b935a766c0..6a682c86efd 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -20,7 +20,10 @@ permissions: jobs: build-and-upload-debs: name: "Build .deb packages and attach to release (Ubuntu 24.04 / noble)" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm + CCACHE_DIR: ${{ github.workspace }}/.cache/ccache steps: - name: Resolve release tag id: tag @@ -36,7 +39,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - submodules: true ref: ${{ steps.tag.outputs.ref }} - name: Verify release tag matches SDK version @@ -57,38 +59,24 @@ jobs: exit 1 fi - - name: Prepare Debian package cache key - id: deb-package-cache-key - shell: bash - run: | - echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" - - - name: Restore Debian package build cache - uses: actions/cache/restore@v4 + - name: Restore CPM and compiler caches + uses: actions/cache@v4 with: path: | - .deb-ccache - build_googleapis_deb - build-deb - key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + .cache/cpm + .cache/ccache + key: deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }} restore-keys: | - ${{ steps.deb-package-cache-key.outputs.prefix }}- - ubuntu-24.04-deb-packages- + deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}- - - name: Build .deb packages in Ubuntu 24.04 container + - name: Build .deb packages with CPack shell: bash run: | - mkdir -p artifacts .deb-ccache build_googleapis_deb build-deb - docker run --rm --network host \ - -e CCACHE_DIR=/source/.deb-ccache \ - -v "$PWD:/source" \ - ubuntu:24.04 \ - bash /source/scripts/build_cpack_deb_packages.sh /source/artifacts + ./scripts/build_cpack_deb_packages.sh artifacts - - name: Smoke-test generated .deb packages + - name: Test .deb packages shell: bash - run: | - ./scripts/test_deb_packages.sh artifacts + run: ./scripts/test_deb_packages.sh artifacts - name: List built artifacts shell: bash diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index 90c393f2798..28a4769e15d 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -66,7 +66,6 @@ jobs: with: path: sdk-current fetch-depth: 0 - submodules: true - name: Determine baseline commit id: baseline @@ -95,7 +94,6 @@ jobs: ref: ${{ steps.baseline.outputs.sha }} path: sdk-baseline fetch-depth: 1 - submodules: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 54e79c108dd..9300e315441 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -14,39 +14,41 @@ concurrency: jobs: build-and-unit: concurrency: - group: build-and-unit-${{ github.ref }}-ubuntu-22.04-${{ matrix.compiler }} + group: build-and-unit-${{ github.ref }}-ubuntu-24.04-${{ matrix.compiler }} cancel-in-progress: true strategy: fail-fast: false matrix: compiler: [clang, gcc] env: - OS: ubuntu-22.04 - runs-on: ubuntu-22.04 + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm + runs-on: ubuntu-24.04 steps: - name: Checkout PR uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' + - name: Restore CPM sources + uses: actions/cache@v4 with: - submodules: true + path: .cache/cpm + key: cpm-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }} - name: Prepare ccache timestamp id: ccache_cache_timestamp shell: bash run: | echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: ~/.ccache - key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + key: ubuntu-24.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | - ubuntu-22.04-ccache-${{ matrix.compiler }}- + ubuntu-24.04-ccache-${{ matrix.compiler }}- - name: Install dependencies uses: ./.github/actions/prepare_vm - name: Build @@ -69,14 +71,12 @@ jobs: --exclude='.ninja_deps' \ --exclude='.ninja_log' \ -czf "integration-build-${{ matrix.compiler }}.tar.gz" . - tar -C "$HOME" -czf "integration-deps-${{ matrix.compiler }}.tar.gz" ydb_deps - name: Upload integration build uses: actions/upload-artifact@v4 with: name: integration-${{ matrix.compiler }} path: | integration-build-${{ matrix.compiler }}.tar.gz - integration-deps-${{ matrix.compiler }}.tar.gz retention-days: 3 integration: @@ -84,7 +84,7 @@ jobs: concurrency: group: integration-${{ github.ref }}-${{ matrix.ydb-version }}-${{ matrix.compiler }} cancel-in-progress: true - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -109,13 +109,10 @@ jobs: uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' - with: - submodules: true - name: Install runtime dependencies uses: ./.github/actions/prepare_vm with: @@ -130,7 +127,6 @@ jobs: run: | mkdir -p build tar -C build -xzf "integration-build-${{ matrix.compiler }}.tar.gz" - tar -C "$HOME" -xzf "integration-deps-${{ matrix.compiler }}.tar.gz" - name: Test shell: bash run: | @@ -154,20 +150,24 @@ jobs: fail-fast: false matrix: compiler: [clang, gcc] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm steps: - name: Checkout PR uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' + - name: Restore CPM sources + uses: actions/cache@v4 with: - submodules: true + path: .cache/cpm + key: cpm-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }} - name: Install dependencies uses: ./.github/actions/prepare_vm @@ -179,12 +179,12 @@ jobs: echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - name: Restore cache files - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + key: ubuntu-24.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | - ${{ runner.os }}-ccache-${{ matrix.compiler }}- + ubuntu-24.04-ccache-${{ matrix.compiler }}- - name: Build and Install SDK shell: bash @@ -209,66 +209,35 @@ jobs: concurrency: group: deb-packages-${{ github.ref }} cancel-in-progress: true - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm + CCACHE_DIR: ${{ github.workspace }}/.cache/ccache steps: - name: Checkout PR uses: actions/checkout@v4 if: github.event.pull_request.head.sha != '' with: - submodules: true ref: ${{ github.event.pull_request.head.sha }} - name: Checkout uses: actions/checkout@v4 if: github.event.pull_request.head.sha == '' - with: - submodules: true - - - name: Prepare Debian package cache key - id: deb-package-cache-key - shell: bash - run: | - echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" - - - name: Validate dpkg-buildpackage - shell: bash - run: | - ./scripts/test_dpkg_buildpackage.sh - - - name: Restore Debian package build cache - id: deb-package-cache - uses: actions/cache/restore@v4 + - name: Restore CPM and compiler caches + uses: actions/cache@v4 with: path: | - .deb-ccache - build_googleapis_deb - build-deb - key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + .cache/cpm + .cache/ccache + key: deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }} restore-keys: | - ${{ steps.deb-package-cache-key.outputs.prefix }}- - ubuntu-24.04-deb-packages- + deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}- - name: Build CPack .deb packages shell: bash run: | - mkdir -p .deb-ccache build_googleapis_deb build-deb - docker run --rm --network host \ - -e CCACHE_DIR=/source/.deb-ccache \ - -v "$PWD:/source" \ - ubuntu:24.04 \ - bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb + ./scripts/build_cpack_deb_packages.sh build-deb/packages - - name: Smoke-test generated .deb packages + - name: Test CPack .deb packages shell: bash - run: | - ./scripts/test_deb_packages.sh build-deb - - - name: Save Debian package build cache - if: github.event_name != 'pull_request' && success() && steps.deb-package-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - .deb-ccache - build_googleapis_deb - build-deb - key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + run: ./scripts/test_deb_packages.sh build-deb/packages diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml index e5864270b4d..70714a27e28 100644 --- a/.github/workflows/warmup_cache.yaml +++ b/.github/workflows/warmup_cache.yaml @@ -1,127 +1,72 @@ name: Warm up cache -on: +on: schedule: - cron: "0 4 * * *" workflow_dispatch: jobs: main: - name: Build YDB C++ SDK and cache artifacts - runs-on: ubuntu-22.04 + name: Warm CPM and ccache (${{ matrix.compiler }}) + runs-on: ubuntu-24.04 strategy: matrix: compiler: [clang, gcc] + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: true - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: bash - run: | - echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" - - name: Restore cache files - id: ccache-restore - uses: actions/cache/restore@v4 - with: - path: ~/.ccache - key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} - restore-keys: | - ubuntu-22.04-ccache-${{ matrix.compiler }}- - - name: Install dependencies - uses: ./.github/actions/prepare_vm - - name: Build - uses: ./.github/actions/build - with: - compiler: ${{ matrix.compiler }} - - name: Save cache files - if: steps.ccache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: ~/.ccache - key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: .cache/cpm + key: cpm-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }} + - uses: actions/cache@v4 + with: + path: ~/.ccache + key: ubuntu-24.04-ccache-${{ matrix.compiler }}-${{ github.run_id }} + restore-keys: | + ubuntu-24.04-ccache-${{ matrix.compiler }}- + - uses: ./.github/actions/prepare_vm + - uses: ./.github/actions/build + with: + compiler: ${{ matrix.compiler }} coverage: - name: Build coverage binaries and cache artifacts - runs-on: ubuntu-22.04 + name: Warm coverage compiler cache + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: true - - name: Prepare coverage cache key - id: coverage-cache-key - shell: bash - run: | - echo "prefix=ubuntu-22.04-coverage-gcc-${{ hashFiles('CMakePresets.json', '**/CMakeLists.txt', 'cmake/**') }}" >> "$GITHUB_OUTPUT" - - name: Restore coverage build cache - id: coverage-build-cache - uses: actions/cache/restore@v4 - with: - path: build - key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.coverage-cache-key.outputs.prefix }}- - - name: Install dependencies - uses: ./.github/actions/prepare_vm - - name: Configure and build with coverage - shell: bash - run: | - mkdir -p build - cmake --preset coverage-test-gcc - cmake --build build -j"$(nproc)" - find build -type f \( -name '*.gcda' -o -name '*.gcov' \) -delete - rm -rf build/coverage - cmake -E remove_directory build/util/system/testing_out_stuff - - name: Save coverage build cache - if: steps.coverage-build-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: build - key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + .cache/cpm + ~/.ccache + key: coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }} + restore-keys: | + coverage-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}- + - uses: ./.github/actions/prepare_vm + - name: Configure and build coverage binaries + run: | + cmake --preset coverage-test-gcc + cmake --build build --parallel deb-packages: - name: Build Debian packages and cache artifacts - runs-on: ubuntu-latest + name: Warm Ubuntu 24.04 DEB caches + runs-on: ubuntu-24.04 + env: + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cache/cpm + CCACHE_DIR: ${{ github.workspace }}/.cache/ccache steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: true - - name: Prepare Debian package cache key - id: deb-package-cache-key - shell: bash - run: | - echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" - - name: Restore Debian package build cache - id: deb-package-cache - uses: actions/cache/restore@v4 - with: - path: | - .deb-ccache - build_googleapis_deb - build-deb - key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - ${{ steps.deb-package-cache-key.outputs.prefix }}- - ubuntu-24.04-deb-packages- - - name: Build CPack .deb packages - shell: bash - run: | - mkdir -p .deb-ccache build_googleapis_deb build-deb - docker run --rm --network host \ - -e CCACHE_DIR=/source/.deb-ccache \ - -v "$PWD:/source" \ - ubuntu:24.04 \ - bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb - - name: Save Debian package build cache - if: steps.deb-package-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - .deb-ccache - build_googleapis_deb - build-deb - key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + .cache/cpm + .cache/ccache + key: deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}-${{ github.run_id }} + restore-keys: | + deb-ubuntu-24.04-${{ hashFiles('cmake/CPM.cmake', 'cmake/dependencies.cmake') }}- + - name: Build CPack DEBs + run: ./scripts/build_cpack_deb_packages.sh build-deb/packages diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d5951cffaab..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,15 +0,0 @@ -[submodule "third_party/api-common-protos"] - path = third_party/api-common-protos - url = https://github.com/googleapis/api-common-protos.git -[submodule "third_party/FastLZ"] - path = third_party/FastLZ - url = https://github.com/ariya/FastLZ -[submodule "third_party/aklomp-base64"] - path = third_party/aklomp-base64 - url = https://github.com/aklomp/base64.git -[submodule "third_party/jwt-cpp"] - path = third_party/jwt-cpp - url = https://github.com/Thalhammer/jwt-cpp.git -[submodule "third_party/opentelemetry-cpp"] - path = third_party/opentelemetry-cpp - url = https://github.com/open-telemetry/opentelemetry-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 72b66aad079..e28d30dca43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.22) file(READ "src/version.h" YDB_SDK_VERSION_FILE_RAW) string(REGEX MATCH "YDB_SDK_VERSION = \"([0-9]+\\.[0-9]+\\.[0-9]+)\"" _ ${YDB_SDK_VERSION_FILE_RAW}) @@ -7,16 +7,29 @@ message(STATUS "YDB C++ SDK version: ${YDB_SDK_VERSION}") project(YDB-CPP-SDK VERSION ${YDB_SDK_VERSION} LANGUAGES C CXX ASM) -option(YDB_SDK_INSTALL "Install YDB C++ SDK" Off) -option(YDB_SDK_TESTS "Build YDB C++ SDK tests" Off) +option(YDB_SDK_INSTALL "Install the YDB C++ SDK and its CPM-managed dependencies" Off) +option(YDB_SDK_TESTS "Build all YDB C++ SDK test binaries" Off) option(YDB_SDK_EXAMPLES "Build YDB C++ SDK examples" On) -option(YDB_SDK_ENABLE_OTEL_METRICS "Build OpenTelemetry metrics plugin" Off) -option(YDB_SDK_ENABLE_OTEL_TRACE "Build OpenTelemetry trace plugin" Off) -option(YDB_CPP_SDK_SLO_USE_INSTALLED_SDK "Build only SLO workloads against an installed ydb-cpp-sdk package" Off) -set(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET "" CACHE STRING "Name of cmake target preparing google common proto library") -option(YDB_SDK_USE_RAPID_JSON "Search for rapid json library in system" ON) +option(YDB_SDK_ENABLE_OTEL_METRICS "Build the OpenTelemetry metrics plugin" Off) +option(YDB_SDK_ENABLE_OTEL_TRACE "Build the OpenTelemetry tracing plugin" Off) +option(YDB_CPP_SDK_SLO_USE_INSTALLED_SDK "Build only SLO workloads against one installed SDK prefix" Off) +set(YDB_SDK_DEPENDENCY_MODE "CPM" CACHE STRING + "Dependency provider: CPM (default, pinned sources) or SYSTEM (Ubuntu DEB builds)") +set_property(CACHE YDB_SDK_DEPENDENCY_MODE PROPERTY STRINGS CPM SYSTEM) +string(TOUPPER "${YDB_SDK_DEPENDENCY_MODE}" YDB_SDK_DEPENDENCY_MODE) +if(NOT YDB_SDK_DEPENDENCY_MODE MATCHES "^(CPM|SYSTEM)$") + message(FATAL_ERROR + "YDB_SDK_DEPENDENCY_MODE must be CPM or SYSTEM, got '${YDB_SDK_DEPENDENCY_MODE}'") +endif() +set(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET "" CACHE STRING + "Advanced override: existing target that provides Google API common protos") +mark_as_advanced(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET) +option(YDB_SDK_USE_RAPID_JSON "Enable JSON support through RapidJSON" ON) set(BUILD_SHARED_LIBS Off) +if(YDB_SDK_INSTALL) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) +endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED On) set(CMAKE_CXX_EXTENSIONS On) @@ -50,38 +63,58 @@ include(cmake/install.cmake) include(cmake/common.cmake) include(cmake/ccache.cmake) include(cmake/protobuf.cmake) -include(cmake/external_libs.cmake) - -set(YDB_SDK_TESTING_INCLUDED OFF) -if (YDB_SDK_TESTS) - include(cmake/testing.cmake) - set(YDB_SDK_TESTING_INCLUDED ON) -endif() if (YDB_CPP_SDK_SLO_USE_INSTALLED_SDK) - if (NOT YDB_SDK_TESTING_INCLUDED) - include(cmake/testing.cmake) - endif() + set(YDB_SDK_DEPENDENCIES_HDR_ONLY ON) + include(cmake/dependencies.cmake) + include(cmake/testing.cmake) - find_package(ydb-cpp-sdk REQUIRED COMPONENTS Table Iam) + find_package(ydb-cpp-sdk REQUIRED COMPONENTS + Coordination + Driver + FederatedTopic + Iam + OpenTelemetryMetrics + Operation + Result + Scheme + SolomonStats + Table + Topic + Types + ) add_subdirectory(tools) add_subdirectory(contrib/libs) + _ydb_sdk_prepare_imported_libc_compat(contrib-libs-libc_compat) add_subdirectory(src/library/string_utils/base64) add_subdirectory(library/cpp) add_subdirectory(util) + target_sources(yutil PRIVATE "${YDB_SDK_SOURCE_DIR}/cmake/tstring_abi.cpp") add_subdirectory(tests/slo_workloads) return() endif() +include(cmake/external_libs.cmake) + +if (YDB_SDK_TESTS) + include(cmake/testing.cmake) +endif() + +set(_ydb_sdk_tests "${YDB_SDK_TESTS}") +set(YDB_SDK_TESTS OFF) add_subdirectory(tools) add_subdirectory(contrib/libs) +_ydb_sdk_prepare_imported_libc_compat(contrib-libs-libc_compat) add_subdirectory(library/cpp) add_subdirectory(include/ydb-cpp-sdk/client) add_subdirectory(src) add_subdirectory(util) +target_sources(yutil PRIVATE "${YDB_SDK_SOURCE_DIR}/cmake/tstring_abi.cpp") add_subdirectory(plugins) +set(YDB_SDK_TESTS "${_ydb_sdk_tests}") +unset(_ydb_sdk_tests) #_ydb_sdk_validate_public_headers() @@ -154,6 +187,7 @@ if (YDB_SDK_INSTALL) ) if (TargetName STREQUAL "libydb-cpp") set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp) + _ydb_sdk_prepare_imported_libc_compat(${TargetName}) endif() target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs}) if (NOT TargetName STREQUAL "libydb-cpp") diff --git a/CMakePresets.json b/CMakePresets.json index 2061ba6b78f..74a2bdf5517 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -8,10 +8,11 @@ "configurePresets": [ { "name": "base", - "displayName": "Generic Config", + "displayName": "CPM-first generic configuration", + "description": "Fetch pinned libraries through CPM; CPM_SOURCE_CACHE may point at a reusable or offline cache.", "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", - "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/brotli" + "YDB_SDK_DEPENDENCY_MODE": "CPM" }, "generator": "Ninja", "binaryDir": "${sourceDir}/build" @@ -37,9 +38,7 @@ "hidden": true, "cacheVariables": { "CMAKE_CXX_COMPILER": "clang++", - "CMAKE_C_COMPILER": "clang", - "CMAKE_EXE_LINKER_FLAGS": "-fuse-ld=lld", - "CMAKE_SHARED_LINKER_FLAGS": "-fuse-ld=lld" + "CMAKE_C_COMPILER": "clang" } }, { @@ -63,12 +62,13 @@ { "name": "release-clang", "inherits": ["release-base", "clang-toolchain"], - "displayName": "Default Release Config (Clang)" + "displayName": "CPM Release (Clang)" }, { "name": "package-deb-clang", "inherits": ["release-base", "clang-toolchain"], - "displayName": "Package Config (Clang) for .deb", + "displayName": "Ubuntu 24.04 CPack DEB (system libraries)", + "description": "Build all four SDK DEB components against Ubuntu package dependencies.", "binaryDir": "${sourceDir}/build-deb", "cacheVariables": { "YDB_SDK_INSTALL": "ON", @@ -76,7 +76,7 @@ "YDB_SDK_ENABLE_OTEL_TRACE": "ON", "YDB_SDK_EXAMPLES": "OFF", "YDB_SDK_TESTS": "OFF", - "YDB_SDK_USE_SYSTEM_GOOGLEAPIS": "ON", + "YDB_SDK_DEPENDENCY_MODE": "SYSTEM", "BUILD_SHARED_LIBS": "OFF", "CMAKE_INSTALL_PREFIX": "/usr/share/yandex", "CMAKE_PREFIX_PATH": "/usr/share/yandex", @@ -87,37 +87,37 @@ { "name": "release-gcc", "inherits": ["release-base", "gcc-toolchain"], - "displayName": "Default Release Config (GCC)" + "displayName": "CPM Release (GCC)" }, { "name": "release-test-clang", "inherits": ["release-base", "clang-toolchain", "test"], - "displayName": "Default Release Test Config (Clang)" + "displayName": "CPM Release with all tests and examples (Clang)" }, { "name": "release-test-gcc", "inherits": ["release-base", "gcc-toolchain", "test"], - "displayName": "Default Release Test Config (GCC)" + "displayName": "CPM Release with all tests and examples (GCC)" }, { "name": "debug-clang", "inherits": ["debug-base", "clang-toolchain"], - "displayName": "Default Debug Config (Clang)" + "displayName": "CPM Debug (Clang)" }, { "name": "debug-gcc", "inherits": ["debug-base", "gcc-toolchain"], - "displayName": "Default Debug Config (GCC)" + "displayName": "CPM Debug (GCC)" }, { "name": "debug-test-clang", "inherits": ["debug-base", "clang-toolchain", "test"], - "displayName": "Default Debug Test Config (Clang)" + "displayName": "CPM Debug with all tests and examples (Clang)" }, { "name": "debug-test-gcc", "inherits": ["debug-base", "gcc-toolchain", "test"], - "displayName": "Default Debug Test Config (GCC)" + "displayName": "CPM Debug with all tests and examples (GCC)" }, { "name": "coverage-base", @@ -132,7 +132,7 @@ { "name": "coverage-test-gcc", "inherits": ["coverage-base", "gcc-toolchain"], - "displayName": "Coverage Test Config (GCC)" + "displayName": "CPM coverage with all tests and examples (GCC)" } ], "buildPresets": [ diff --git a/README.md b/README.md index 41d1790d193..d89af4311a3 100644 --- a/README.md +++ b/README.md @@ -7,150 +7,47 @@ ### Prerequisites - cmake 3.22+ -- clang 18+ +- a current C++20 compiler (GCC or Clang) - git 2.20+ - ninja 1.10+ - ragel - yasm -- protoc - -### Library dependencies - -- gRPC -- protobuf - OpenSSL - Iconv - IDN -- rapidjson -- xxhash -- zlib -- zstd -- lz4 -- snappy 1.1.8+ -- base64 -- brotli 1.1.0+ -- double-conversion -- jwt-cpp - -### Runtime requirements - -- libidn11-dev (IDN) -- libiconv (Iconv) - -### Testing requirements - -- gtest -- gmock ### Install dependencies -The standalone dependency bundle is compatible with Ubuntu 24.04 and uses -gRPC 1.60.2 to match the imported YDB sources. Its protobuf and Abseil pins -match the dependency set published with that gRPC release: +CPM fetches the pinned library dependencies during CMake configure. The only +packages to install manually are the tools and platform prerequisites above. +The authoritative pins are in `cmake/dependencies.cmake`. -| Dependency | Version | -|------------|---------| -| Abseil | 20230802.0 | -| protobuf | 25.0 | -| gRPC | 1.60.2 | - -These pins are shared by regular CI builds, SLO workload images, and the -development container. +Ubuntu 24.04: ```bash sudo apt-get -y update -sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 - -wget https://apt.llvm.org/llvm.sh -chmod u+x llvm.sh -sudo ./llvm.sh 18 - -# Install abseil-cpp -wget -O abseil-cpp-20230802.0.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz -tar -xvzf abseil-cpp-20230802.0.tar.gz -cd abseil-cpp-20230802.0 -mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. -cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/absl -cd ../../ - -# Install protobuf -wget -O protobuf-25.0.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v25.0.tar.gz -tar -xvzf protobuf-25.0.tar.gz -cd protobuf-25.0 -mkdir build && cd build -cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl" -DCMAKE_BUILD_TYPE=Release \ - -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. -cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/protobuf -cd ../../ - -# Install gRPC -wget -O grpc-1.60.2.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v1.60.2.tar.gz -tar -xvzf grpc-1.60.2.tar.gz && cd grpc-1.60.2 -mkdir build && cd build -cmake -G Ninja -DCMAKE_PREFIX_PATH="$HOME/ydb_deps/absl;$HOME/ydb_deps/protobuf" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ - -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. -cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/grpc -cd ../../ - -# Install base64 -wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz -tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2 -mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. -cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/base64 -cd ../../ - -# Install brotli -wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz -tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 -mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="$HOME/ydb_deps/brotli" .. -cmake --build . --config Release -cmake --install . --config Release -cd ../../ - -# Install jwt-cpp -wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz -tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0 -mkdir build && cd build -cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. -cmake --build . --config Release -cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp -cd ../../ +sudo apt-get -y install build-essential ca-certificates ccache clang cmake git \ + libidn11-dev libssl-dev lld ninja-build pkg-config python3 ragel yasm ``` -### Create the work directory +Fedora 43: ```bash -mkdir ~/ydbwork && cd ~/ydbwork -mkdir build +sudo dnf install -y ccache cmake gcc gcc-c++ git libidn-devel \ + ninja-build openssl-devel openssl-devel-engine pkgconf-pkg-config python3 ragel yasm ``` -### Install ccache - -Install `ccache` into `/usr/local/bin/`. The recommended version is `4.8.1` or above, the minimum required version is `4.7`. +macOS 14: ```bash -(V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \ -sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache) +xcode-select --install # if the Command Line Tools are not installed yet +brew install ccache cmake git libidn ninja openssl@3 python ragel yasm ``` ### Clone the ydb-cpp-sdk repository ```bash -git clone --recurse-submodules https://github.com/ydb-platform/ydb-cpp-sdk.git +git clone https://github.com/ydb-platform/ydb-cpp-sdk.git ``` ### Configure @@ -162,10 +59,18 @@ cd ydb-cpp-sdk cmake --preset $sdk_configure_preset ``` +`YDB_SDK_DEPENDENCY_MODE` defaults to `CPM`. Set `CPM_SOURCE_CACHE` to a +reusable directory before configuring; a fully populated cache can be reused +offline with `FETCHCONTENT_FULLY_DISCONNECTED=ON`. `SYSTEM` mode is intended +only for Ubuntu DEB builds. + +With the Dev Container CLI, `devcontainer up --workspace-folder .` creates the +development container and configures the test build through the same CPM path. + ### Build ```bash -cmake --build --preset $sdk_configure_preset +cmake --build build --parallel ``` ### Build `.deb` packages @@ -175,54 +80,25 @@ The SDK can be packaged as Debian development packages with CPack. The complete - `yandex-googleapis-api-common-protos` — generated API Common Protos headers and static library, required by `libydb-cpp-dev`; - `libydb-cpp-dev` — core SDK static library, public headers and CMake package files; - `libydb-cpp-iam-dev` — IAM credentials plugin; -- `libydb-cpp-otel-metrics-dev` — OpenTelemetry metrics plugin (includes vendored opentelemetry-cpp); +- `libydb-cpp-otel-metrics-dev` — OpenTelemetry metrics plugin; - `libydb-cpp-otel-tracing-dev` — OpenTelemetry tracing plugin (requires `libydb-cpp-otel-metrics-dev` for OTel headers/libs). -The Debian packaging flow is intended for Ubuntu 24.04. Initialize the required submodules before building: - -```bash -git submodule update --init --recursive -``` - -The packaging helper first builds `yandex-googleapis-api-common-protos` from the -vendored API Common Protos using Ubuntu 24.04's `protobuf-compiler` and -`libprotobuf-dev`. It installs that package in the build container before -building the SDK, so both packages use the same distro protobuf ABI. -OpenTelemetry plugins use the vendored `third_party/opentelemetry-cpp` -submodule (v1.26.0, matching the YDB monorepo pin). - -To build the complete `.deb` package set with the same containerized flow used -by CI and release publishing (Docker is required): - -```bash -mkdir -p build-deb .deb-ccache -docker run --rm --network host \ - -e CCACHE_DIR=/source/.deb-ccache \ - -v "$PWD:/source" \ - ubuntu:24.04 \ - bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb -``` - -The generated `.deb` files are placed into `build-deb/`. - -To prepare Debian source-package metadata, run: +The CPack-only packaging flow is intended for Ubuntu 24.04. It builds and +installs the Google common-protos package before packaging the four SDK +components, so all five packages use the distro protobuf ABI: ```bash -./scripts/generate-debian-directory.sh +./scripts/build_cpack_deb_packages.sh build-deb/packages ``` -This updates `debian/changelog` from `src/version.h` and the current Git commit. The `debian/` directory contains package metadata and install manifests for `dpkg-buildpackage`/PPA builds. - -To validate a binary Debian package build in an Ubuntu 24.04 Docker container: - -```bash -./scripts/test_dpkg_buildpackage.sh -``` +The generated `.deb` files are placed into `build-deb/packages/` and install +under `/usr/share/yandex`. The IAM and OTel packages require the matching core +version; tracing additionally requires the matching metrics package. To smoke-test generated `.deb` packages with the sample consumer project: ```bash -./scripts/test_deb_packages.sh build-deb +./scripts/test_deb_packages.sh build-deb/packages ``` ### Install from GitHub releases @@ -235,7 +111,7 @@ GitHub release. Download the assets and install them with APT: TAG= BASE="https://github.com/ydb-platform/ydb-cpp-sdk/releases/download/${TAG}" -wget "${BASE}/yandex-googleapis-api-common-protos-1.0.0-Linux.deb" +wget "${BASE}/yandex-googleapis-api-common-protos_1.0.0_amd64.deb" wget "${BASE}/libydb-cpp-dev_${TAG#v}_amd64.deb" # Optional plugins: wget "${BASE}/libydb-cpp-iam-dev_${TAG#v}_amd64.deb" @@ -244,7 +120,7 @@ wget "${BASE}/libydb-cpp-otel-tracing-dev_${TAG#v}_amd64.deb" sudo apt-get update sudo apt-get install -y \ - ./yandex-googleapis-api-common-protos-*.deb \ + ./yandex-googleapis-api-common-protos_*.deb \ ./libydb-cpp-dev_*.deb ./libydb-cpp-iam-dev_*.deb \ ./libydb-cpp-otel-metrics-dev_*.deb ./libydb-cpp-otel-tracing-dev_*.deb ``` diff --git a/cmake/CPM.cmake b/cmake/CPM.cmake new file mode 100644 index 00000000000..a6fc1127cf3 --- /dev/null +++ b/cmake/CPM.cmake @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: Copyright (c) 2019-2025 Lars Melchior and contributors + +# Keep the bootstrap small and reviewable while pinning the downloaded CPM +# implementation by both version and digest. +set(CPM_DOWNLOAD_VERSION 0.42.0) +set(CPM_DOWNLOAD_SHA256 + 2020b4fc42dba44817983e06342e682ecfc3d2f484a581f11cc5731fbe4dce8a) + +if(CPM_SOURCE_CACHE) + set(CPM_DOWNLOAD_LOCATION + "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +elseif(DEFINED ENV{CPM_SOURCE_CACHE}) + set(CPM_DOWNLOAD_LOCATION + "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +else() + set(CPM_DOWNLOAD_LOCATION + "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") +endif() + +get_filename_component(CPM_DOWNLOAD_LOCATION "${CPM_DOWNLOAD_LOCATION}" ABSOLUTE) +get_filename_component(_cpm_download_directory "${CPM_DOWNLOAD_LOCATION}" DIRECTORY) +file(MAKE_DIRECTORY "${_cpm_download_directory}") + +set(_cpm_download_required ON) +if(EXISTS "${CPM_DOWNLOAD_LOCATION}") + file(SHA256 "${CPM_DOWNLOAD_LOCATION}" _cpm_download_sha256) + if(_cpm_download_sha256 STREQUAL CPM_DOWNLOAD_SHA256) + set(_cpm_download_required OFF) + endif() +endif() + +if(_cpm_download_required) + file(DOWNLOAD + "https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake" + "${CPM_DOWNLOAD_LOCATION}" + EXPECTED_HASH "SHA256=${CPM_DOWNLOAD_SHA256}" + TLS_VERIFY ON + ) +endif() + +unset(_cpm_download_required) +unset(_cpm_download_sha256) + +include("${CPM_DOWNLOAD_LOCATION}") diff --git a/cmake/PackSDK.cmake b/cmake/PackSDK.cmake index c204e7a49e3..7981a046f22 100644 --- a/cmake/PackSDK.cmake +++ b/cmake/PackSDK.cmake @@ -21,7 +21,7 @@ set(CPACK_COMPONENTS_ALL libydb-cpp libydb-cpp-iam libydb-cpp-otel-metrics libyd set(CPACK_DEBIAN_LIBYDB_CPP_PACKAGE_NAME "libydb-cpp-dev") set(CPACK_DEBIAN_LIBYDB_CPP_PACKAGE_DEPENDS - "libidn11-dev, libssl-dev, zlib1g-dev, libprotobuf-dev, libgrpc++-dev, libbrotli-dev, liblz4-dev, libzstd-dev, libbz2-dev, libxxhash-dev, libsnappy-dev, libdouble-conversion-dev, libgtest-dev, libre2-dev, libc-ares-dev, rapidjson-dev, yandex-googleapis-api-common-protos" + "libidn11-dev, libssl-dev, zlib1g-dev, libprotobuf-dev, libgrpc++-dev, protobuf-compiler-grpc, libbrotli-dev, liblz4-dev, libzstd-dev, libbz2-dev, libxxhash-dev, libsnappy-dev, libdouble-conversion-dev, libre2-dev, libc-ares-dev, rapidjson-dev, yandex-googleapis-api-common-protos" ) set(CPACK_DEBIAN_LIBYDB_CPP_IAM_PACKAGE_NAME "libydb-cpp-iam-dev") diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index 311a9b8f128..ef68402c465 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -1,7 +1,5 @@ find_program(CCACHE_PATH ccache) -if (YDB_SDK_COVERAGE) - message(STATUS "YDB_SDK_COVERAGE is ON: ccache compiler launchers disabled") -elseif (NOT CCACHE_PATH) +if (NOT CCACHE_PATH) message(AUTHOR_WARNING "Ccache is not found, that will increase the re-compilation time; " "pass -DCCACHE_PATH=path/to/bin to specify the path to the `ccache` binary file." diff --git a/cmake/common.cmake b/cmake/common.cmake index b40eee34324..e32407e16cb 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -2,6 +2,16 @@ find_package(Python3 REQUIRED) +function(_ydb_sdk_prepare_imported_libc_compat TargetName) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND + CMAKE_C_COMPILER_ID STREQUAL "GNU" AND + CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 14) + target_compile_definitions(${TargetName} PRIVATE _GNU_SOURCE) + target_compile_options(${TargetName} PRIVATE + "$<$:SHELL:-include unistd.h>") + endif() +endfunction() + # assumes ToolName is always both the binary and the target name function(get_built_tool_path OutBinPath SrcPath ToolName) set(${OutBinPath} "${YDB_SDK_BINARY_DIR}/${SrcPath}/${ToolName}${CMAKE_EXECUTABLE_SUFFIX}" PARENT_SCOPE) @@ -9,6 +19,12 @@ endfunction() function(target_ragel_lexers TgtName Key Src) SET(RAGEL_BIN ragel${CMAKE_EXECUTABLE_SUFFIX}) + set(RagelArgs ${ARGN}) + execute_process(COMMAND ${RAGEL_BIN} --version OUTPUT_VARIABLE RagelVersion) + if(RagelVersion MATCHES "version ([0-9]+)" AND CMAKE_MATCH_1 GREATER_EQUAL 7) + # Ragel 7 selects C/C++ from the executable name and rejects Ragel 6's -C. + list(TRANSFORM RagelArgs REPLACE "^-C" "-") + endif() get_filename_component(OutPath ${Src} NAME_WLE) get_filename_component(SrcDirPath ${Src} DIRECTORY) get_filename_component(OutputExt ${OutPath} EXT) @@ -17,7 +33,7 @@ function(target_ragel_lexers TgtName Key Src) endif() add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${OutPath} - COMMAND Python3::Interpreter ${YDB_SDK_SOURCE_DIR}/scripts/run_tool.py -- ${RAGEL_BIN} ${RAGEL_FLAGS} ${ARGN} -o ${CMAKE_CURRENT_BINARY_DIR}/${OutPath} ${Src} + COMMAND Python3::Interpreter ${YDB_SDK_SOURCE_DIR}/scripts/run_tool.py -- ${RAGEL_BIN} ${RAGEL_FLAGS} ${RagelArgs} -o ${CMAKE_CURRENT_BINARY_DIR}/${OutPath} ${Src} DEPENDS ${YDB_SDK_SOURCE_DIR}/scripts/run_tool.py ${Src} WORKING_DIRECTORY ${SrcDirPath} ) @@ -301,6 +317,7 @@ function(_ydb_sdk_collect_package_target PackageName Tgt) return() endif() + set(public_target ${Tgt}) get_target_property(aliased_target ${Tgt} ALIASED_TARGET) if (aliased_target) set(Tgt ${aliased_target}) @@ -312,6 +329,19 @@ function(_ydb_sdk_collect_package_target PackageName Tgt) return() endif() + if (YDB_SDK_DEPENDENCY_MODE STREQUAL "CPM") + get_target_property(tgt_source_dir ${Tgt} SOURCE_DIR) + get_property(cpm_source_dirs GLOBAL PROPERTY YDB_SDK_CPM_SOURCE_DIRS) + foreach(cpm_source_dir IN LISTS cpm_source_dirs) + cmake_path(IS_PREFIX cpm_source_dir "${tgt_source_dir}" NORMALIZE is_cpm_target) + if (is_cpm_target) + set_property(GLOBAL APPEND PROPERTY + YDB_CPP_${PackageName}_PUBLIC_DEPS ${public_target}) + return() + endif() + endforeach() + endif() + get_property(visited_targets GLOBAL PROPERTY YDB_CPP_${PackageName}_VISITED_TARGETS) if (Tgt IN_LIST visited_targets) return() diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake new file mode 100644 index 00000000000..90c90d83eab --- /dev/null +++ b/cmake/dependencies.cmake @@ -0,0 +1,536 @@ +include_guard(GLOBAL) + +# This file is the authoritative dependency manifest. Keep version changes here +# so presets, containers, workflows, and documentation do not drift apart. +set(YDB_SDK_ABSEIL_VERSION 20240116.0) +set(YDB_SDK_PROTOBUF_VERSION 26.1) +set(YDB_SDK_GRPC_REVISION e55f69cedd0ef7344e0bcb64b5ec9205e6aa4f04) +set(YDB_SDK_GRPC_SHA256 acdf93cc2d116769175712de3606f0ec6a4989f49276ca894b3ee398b1954015) +set(YDB_SDK_BASE64_VERSION 0.5.2) +set(YDB_SDK_BROTLI_VERSION 1.1.0) +set(YDB_SDK_JWT_CPP_VERSION 0.6.0) +set(YDB_SDK_OPENTELEMETRY_VERSION 1.26.0) +set(YDB_SDK_GOOGLETEST_VERSION 1.15.2) +set(YDB_SDK_HDR_HISTOGRAM_VERSION 0.11.8) + +set(YDB_SDK_ZLIB_VERSION 1.3.1) +set(YDB_SDK_XXHASH_VERSION 0.8.3) +set(YDB_SDK_ZSTD_VERSION 1.5.7) +set(YDB_SDK_BZIP2_VERSION 1.0.8) +set(YDB_SDK_LZ4_VERSION 1.10.0) +set(YDB_SDK_SNAPPY_VERSION 1.2.1) +set(YDB_SDK_DOUBLE_CONVERSION_VERSION 3.3.0) +set(YDB_SDK_RAPIDJSON_VERSION 1.1.0) +set(YDB_SDK_CARES_VERSION 1.27.0) +set(YDB_SDK_RE2_VERSION 2023-11-01) +set(YDB_SDK_CURL_VERSION 8.19.0) +set(YDB_SDK_OTEL_PROTO_VERSION 1.8.0) +string(REPLACE "." "_" YDB_SDK_CARES_GIT_VERSION "${YDB_SDK_CARES_VERSION}") + +set(YDB_SDK_GOOGLEAPIS_GIT_TAG 3332dec527759859840a3a2ff108c67a54708130) +set(YDB_SDK_FASTLZ_GIT_TAG 344eb4025f9ae866ebf7a2ec48850f7113a97a42) + +set(CPM_USE_NAMED_CACHE_DIRECTORIES ON CACHE BOOL + "Use stable per-package directories in CPM_SOURCE_CACHE") +set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING + "Minimum CMake policy version accepted by fetched dependencies") +include("${CMAKE_CURRENT_LIST_DIR}/CPM.cmake") + +set(_YDB_SDK_CPM_EXCLUDE_FROM_ALL YES) +set(_YDB_SDK_CPM_DISABLE_INSTALL YES) +if(YDB_SDK_INSTALL) + set(_YDB_SDK_CPM_EXCLUDE_FROM_ALL NO) + set(_YDB_SDK_CPM_DISABLE_INSTALL NO) +endif() + +function(_ydb_sdk_cpm_package_stub package_name contents) + set(_stub_dir "${CMAKE_BINARY_DIR}/cpm-package-stubs/${package_name}") + file(MAKE_DIRECTORY "${_stub_dir}") + file(WRITE "${_stub_dir}/${package_name}Config.cmake" "${contents}\n") + set(${package_name}_DIR "${_stub_dir}" CACHE PATH "CPM package adapter" FORCE) +endfunction() + +function(_ydb_sdk_alias_library alias_name) + if(TARGET "${alias_name}") + return() + endif() + foreach(_candidate IN LISTS ARGN) + if(TARGET "${_candidate}") + add_library("${alias_name}" INTERFACE IMPORTED GLOBAL) + set_property(TARGET "${alias_name}" PROPERTY + INTERFACE_LINK_LIBRARIES "${_candidate}") + return() + endif() + endforeach() + message(FATAL_ERROR "CPM dependency did not provide a target for ${alias_name}") +endfunction() + +function(ydb_sdk_fetch_googleapis) + if(DEFINED YDB_SDK_GOOGLEAPIS_SOURCE_DIR AND + IS_DIRECTORY "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}/google") + set(YDB_SDK_GOOGLEAPIS_SOURCE_DIR "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}" PARENT_SCOPE) + return() + endif() + CPMAddPackage( + NAME google_api_common_protos + GITHUB_REPOSITORY googleapis/api-common-protos + GIT_TAG ${YDB_SDK_GOOGLEAPIS_GIT_TAG} + DOWNLOAD_ONLY YES + ) + set(YDB_SDK_GOOGLEAPIS_SOURCE_DIR "${google_api_common_protos_SOURCE_DIR}" PARENT_SCOPE) +endfunction() + +function(ydb_sdk_fetch_hdr_histogram) + set(_saved_skip_install_rules "${CMAKE_SKIP_INSTALL_RULES}") + set(CMAKE_SKIP_INSTALL_RULES ON) + CPMAddPackage( + NAME hdr_histogram + GITHUB_REPOSITORY HdrHistogram/HdrHistogram_c + GIT_TAG ${YDB_SDK_HDR_HISTOGRAM_VERSION} + EXCLUDE_FROM_ALL YES + OPTIONS + "HDR_HISTOGRAM_BUILD_PROGRAMS OFF" + "HDR_HISTOGRAM_BUILD_SHARED OFF" + "HDR_LOG_REQUIRED OFF" + ) + set(CMAKE_SKIP_INSTALL_RULES "${_saved_skip_install_rules}") +endfunction() + +if(YDB_SDK_DEPENDENCIES_GOOGLEAPIS_ONLY) + ydb_sdk_fetch_googleapis() + return() +endif() + +if(YDB_SDK_DEPENDENCIES_HDR_ONLY) + ydb_sdk_fetch_hdr_histogram() + return() +endif() + +if(YDB_SDK_DEPENDENCY_MODE STREQUAL "SYSTEM") + find_package(Protobuf CONFIG QUIET) + if(NOT Protobuf_FOUND) + find_package(Protobuf MODULE REQUIRED) + endif() + find_package(gRPC 1.41.0 REQUIRED) + find_package(ZLIB REQUIRED) + find_package(xxHash REQUIRED) + find_package(ZSTD REQUIRED) + find_package(BZip2 REQUIRED) + find_package(LZ4 REQUIRED) + find_package(Snappy 1.1.8 REQUIRED) + find_package(Brotli 1.1.0 REQUIRED) + find_package(double-conversion REQUIRED) + if(YDB_SDK_USE_RAPID_JSON) + find_package(RapidJSON REQUIRED) + if(NOT TARGET RapidJSON::RapidJSON) + add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) + target_include_directories(RapidJSON::RapidJSON INTERFACE ${RAPIDJSON_INCLUDE_DIRS}) + endif() + endif() +else() + set(_ydb_sdk_saved_install_component "${CMAKE_INSTALL_DEFAULT_COMPONENT_NAME}") + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) + set(BUILD_SHARED_LIBS OFF) + + CPMAddPackage( + NAME ZLIB + GITHUB_REPOSITORY madler/zlib + GIT_TAG v${YDB_SDK_ZLIB_VERSION} + EXCLUDE_FROM_ALL YES + OPTIONS "ZLIB_BUILD_EXAMPLES OFF" + ) + if(YDB_SDK_INSTALL) + install(TARGETS zlibstatic ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT libydb-cpp) + install(FILES "${ZLIB_SOURCE_DIR}/zlib.h" "${ZLIB_BINARY_DIR}/zconf.h" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT libydb-cpp) + endif() + # Keep build-tree implementation targets out of protobuf/gRPC install + # exports. Consumers resolve this conventional target through FindZLIB. + if(NOT TARGET ZLIB::ZLIB) + add_library(ZLIB::ZLIB INTERFACE IMPORTED GLOBAL) + set_target_properties(ZLIB::ZLIB PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_SOURCE_DIR};${ZLIB_BINARY_DIR}" + INTERFACE_LINK_LIBRARIES zlibstatic) + add_dependencies(ZLIB::ZLIB zlibstatic) + endif() + _ydb_sdk_cpm_package_stub(ZLIB + "set(ZLIB_FOUND TRUE)\nset(ZLIB_VERSION_STRING ${YDB_SDK_ZLIB_VERSION})\nset(ZLIB_LIBRARIES ZLIB::ZLIB)\nset(ZLIB_LIBRARY ZLIB::ZLIB)\nset(ZLIB_INCLUDE_DIRS \"${ZLIB_SOURCE_DIR};${ZLIB_BINARY_DIR}\")\nset(ZLIB_INCLUDE_DIR \"${ZLIB_SOURCE_DIR}\")") + + CPMAddPackage( + NAME abseil-cpp + GITHUB_REPOSITORY abseil/abseil-cpp + GIT_TAG ${YDB_SDK_ABSEIL_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "ABSL_PROPAGATE_CXX_STD ON" + "ABSL_ENABLE_INSTALL ${YDB_SDK_INSTALL}" + "ABSL_BUILD_TESTING OFF" + ) + if(APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # Keep Abseil's architecture flag pairs intact after CMake de-duplication. + foreach(_target IN ITEMS absl_random_internal_randen_hwaes + absl_random_internal_randen_hwaes_impl) + get_target_property(_copts ${_target} COMPILE_OPTIONS) + list(REMOVE_ITEM _copts + -Xarch_x86_64 -maes -msse4.1 -Xarch_arm64 -march=armv8-a+crypto) + list(APPEND _copts + "SHELL:-Xarch_x86_64 -maes" + "SHELL:-Xarch_x86_64 -msse4.1" + "SHELL:-Xarch_arm64 -march=armv8-a+crypto") + set_property(TARGET ${_target} PROPERTY COMPILE_OPTIONS "${_copts}") + endforeach() + endif() + _ydb_sdk_cpm_package_stub(absl "set(absl_FOUND TRUE)") + + CPMAddPackage( + NAME Protobuf + GITHUB_REPOSITORY protocolbuffers/protobuf + GIT_TAG v${YDB_SDK_PROTOBUF_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "protobuf_BUILD_TESTS OFF" + "protobuf_BUILD_EXAMPLES OFF" + "protobuf_BUILD_SHARED_LIBS OFF" + "protobuf_INSTALL ${YDB_SDK_INSTALL}" + "protobuf_ABSL_PROVIDER package" + "protobuf_WITH_ZLIB ON" + "utf8_range_ENABLE_INSTALL ${YDB_SDK_INSTALL}" + ) + set(YDB_SDK_PROTOBUF_SOURCE_DIR "${Protobuf_SOURCE_DIR}") + _ydb_sdk_alias_library(protobuf::libprotobuf libprotobuf) + if(NOT TARGET protobuf::libprotoc AND TARGET libprotoc) + add_library(protobuf::libprotoc ALIAS libprotoc) + endif() + if(NOT TARGET protobuf::protoc AND TARGET protoc) + add_executable(protobuf::protoc ALIAS protoc) + endif() + _ydb_sdk_cpm_package_stub(Protobuf + "set(Protobuf_FOUND TRUE)\nset(Protobuf_VERSION ${YDB_SDK_PROTOBUF_VERSION})\nset(Protobuf_LIBRARIES protobuf::libprotobuf)\nset(Protobuf_PROTOC_EXECUTABLE \"$\")") + + # c-ares 1.27 only initializes this suffix in its shared-library branch. + set(STATIC_SUFFIX _static) + set(_ydb_sdk_saved_runtime_output "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + CPMAddPackage( + NAME c-ares + GITHUB_REPOSITORY c-ares/c-ares + GIT_TAG cares-${YDB_SDK_CARES_GIT_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "CARES_STATIC ON" + "CARES_SHARED OFF" + "CARES_BUILD_TESTS OFF" + "CARES_BUILD_TOOLS OFF" + "CARES_INSTALL ${YDB_SDK_INSTALL}" + ) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${_ydb_sdk_saved_runtime_output}") + unset(STATIC_SUFFIX) + _ydb_sdk_alias_library(c-ares::cares cares_static cares) + _ydb_sdk_cpm_package_stub(c-ares "set(c-ares_FOUND TRUE)") + + set(_ydb_sdk_saved_skip_install_rules "${CMAKE_SKIP_INSTALL_RULES}") + if(NOT YDB_SDK_INSTALL) + set(CMAKE_SKIP_INSTALL_RULES ON) + endif() + CPMAddPackage( + NAME re2 + GITHUB_REPOSITORY google/re2 + GIT_TAG ${YDB_SDK_RE2_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "RE2_BUILD_TESTING OFF" + "RE2_USE_ICU OFF" + ) + set(CMAKE_SKIP_INSTALL_RULES "${_ydb_sdk_saved_skip_install_rules}") + _ydb_sdk_alias_library(re2::re2 re2) + _ydb_sdk_cpm_package_stub(re2 "set(re2_FOUND TRUE)") + + set(gRPC_INSTALL ${YDB_SDK_INSTALL} CACHE BOOL "" FORCE) + set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(gRPC_BUILD_CODEGEN ON CACHE BOOL "" FORCE) + # gRPC copies this property into protoc command arguments without + # filtering an empty INSTALL_INTERFACE entry. + get_target_property(_ydb_sdk_libprotoc_includes libprotoc INTERFACE_INCLUDE_DIRECTORIES) + set_target_properties(libprotoc PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${Protobuf_SOURCE_DIR}/src") + CPMAddPackage( + NAME gRPC + URL https://github.com/grpc/grpc/archive/${YDB_SDK_GRPC_REVISION}.tar.gz + URL_HASH SHA256=${YDB_SDK_GRPC_SHA256} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "gRPC_INSTALL ${YDB_SDK_INSTALL}" + "gRPC_BUILD_TESTS OFF" + "gRPC_BUILD_CODEGEN ON" + "gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF" + "gRPC_BUILD_GRPC_NODE_PLUGIN OFF" + "gRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN OFF" + "gRPC_BUILD_GRPC_PHP_PLUGIN OFF" + "gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF" + "gRPC_BUILD_GRPC_RUBY_PLUGIN OFF" + "gRPC_ZLIB_PROVIDER package" + "gRPC_CARES_PROVIDER package" + "gRPC_RE2_PROVIDER package" + "gRPC_SSL_PROVIDER package" + "gRPC_PROTOBUF_PROVIDER package" + "gRPC_ABSL_PROVIDER package" + ) + if(_ydb_sdk_libprotoc_includes) + set_target_properties(libprotoc PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_ydb_sdk_libprotoc_includes}") + else() + set_target_properties(libprotoc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") + endif() + if(TARGET grpc++ AND NOT TARGET gRPC::grpc++) + add_library(gRPC::grpc++ ALIAS grpc++) + endif() + if(TARGET grpc_cpp_plugin AND NOT TARGET gRPC::grpc_cpp_plugin) + add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) + endif() + + CPMAddPackage( + NAME Brotli + GITHUB_REPOSITORY google/brotli + GIT_TAG v${YDB_SDK_BROTLI_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS "BROTLI_DISABLE_TESTS ON" "BROTLI_BUNDLED_MODE OFF" + ) + _ydb_sdk_alias_library(Brotli::common brotlicommon-static brotlicommon) + _ydb_sdk_alias_library(Brotli::dec brotlidec-static brotlidec) + _ydb_sdk_alias_library(Brotli::enc brotlienc-static brotlienc) + + CPMAddPackage( + NAME xxHash + GITHUB_REPOSITORY Cyan4973/xxHash + GIT_TAG v${YDB_SDK_XXHASH_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + SOURCE_SUBDIR cmake_unofficial + OPTIONS "XXHASH_BUILD_XXHSUM OFF" "XXHASH_BUILD_ENABLE_INLINE_API OFF" + ) + _ydb_sdk_alias_library(xxHash::xxHash xxhash xxhash_static) + if(YDB_SDK_INSTALL) + install(TARGETS xxhash ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT libydb-cpp) + install(FILES "${xxHash_SOURCE_DIR}/xxhash.h" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT libydb-cpp) + endif() + + CPMAddPackage( + NAME zstd + GITHUB_REPOSITORY facebook/zstd + GIT_TAG v${YDB_SDK_ZSTD_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + SOURCE_SUBDIR build/cmake + OPTIONS + "ZSTD_BUILD_PROGRAMS OFF" + "ZSTD_BUILD_TESTS OFF" + "ZSTD_BUILD_SHARED OFF" + "ZSTD_BUILD_STATIC ON" + ) + _ydb_sdk_alias_library(ZSTD::ZSTD libzstd_static) + + CPMAddPackage( + NAME bzip2 + GITHUB_REPOSITORY libarchive/bzip2 + GIT_TAG bzip2-${YDB_SDK_BZIP2_VERSION} + DOWNLOAD_ONLY YES + ) + add_library(ydb_sdk_bzip2 STATIC + "${bzip2_SOURCE_DIR}/blocksort.c" + "${bzip2_SOURCE_DIR}/huffman.c" + "${bzip2_SOURCE_DIR}/crctable.c" + "${bzip2_SOURCE_DIR}/randtable.c" + "${bzip2_SOURCE_DIR}/compress.c" + "${bzip2_SOURCE_DIR}/decompress.c" + "${bzip2_SOURCE_DIR}/bzlib.c") + set_target_properties(ydb_sdk_bzip2 PROPERTIES OUTPUT_NAME bz2) + target_include_directories(ydb_sdk_bzip2 PUBLIC + $ + $) + _ydb_sdk_alias_library(BZip2::BZip2 ydb_sdk_bzip2) + if(YDB_SDK_INSTALL) + install(TARGETS ydb_sdk_bzip2 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT libydb-cpp) + install(FILES "${bzip2_SOURCE_DIR}/bzlib.h" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT libydb-cpp) + endif() + + CPMAddPackage( + NAME lz4 + GITHUB_REPOSITORY lz4/lz4 + GIT_TAG v${YDB_SDK_LZ4_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + SOURCE_SUBDIR build/cmake + OPTIONS + "LZ4_BUILD_CLI OFF" + "LZ4_BUILD_LEGACY_LZ4C OFF" + "BUILD_SHARED_LIBS OFF" + ) + _ydb_sdk_alias_library(LZ4::LZ4 lz4_static lz4) + if(YDB_SDK_INSTALL) + install(TARGETS lz4_static ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT libydb-cpp) + install(FILES + "${lz4_SOURCE_DIR}/lib/lz4.h" + "${lz4_SOURCE_DIR}/lib/lz4frame.h" + "${lz4_SOURCE_DIR}/lib/lz4hc.h" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT libydb-cpp) + endif() + + CPMAddPackage( + NAME Snappy + GITHUB_REPOSITORY google/snappy + GIT_TAG ${YDB_SDK_SNAPPY_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "SNAPPY_BUILD_TESTS OFF" + "SNAPPY_BUILD_BENCHMARKS OFF" + "SNAPPY_INSTALL ON" + ) + _ydb_sdk_alias_library(Snappy::snappy snappy) + + CPMAddPackage( + NAME double-conversion + GITHUB_REPOSITORY google/double-conversion + GIT_TAG v${YDB_SDK_DOUBLE_CONVERSION_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS "BUILD_TESTING OFF" + ) + _ydb_sdk_alias_library(double-conversion::double-conversion double-conversion) + + if(YDB_SDK_USE_RAPID_JSON) + CPMAddPackage( + NAME RapidJSON + GITHUB_REPOSITORY Tencent/rapidjson + GIT_TAG v${YDB_SDK_RAPIDJSON_VERSION} + EXCLUDE_FROM_ALL YES + OPTIONS + "RAPIDJSON_BUILD_DOC OFF" + "RAPIDJSON_BUILD_EXAMPLES OFF" + "RAPIDJSON_BUILD_TESTS OFF" + "RAPIDJSON_BUILD_THIRDPARTY_GTEST OFF" + ) + if(NOT TARGET RapidJSON::RapidJSON) + add_library(RapidJSON::RapidJSON INTERFACE IMPORTED GLOBAL) + target_include_directories(RapidJSON::RapidJSON INTERFACE + "$") + endif() + _ydb_sdk_cpm_package_stub(RapidJSON + "get_filename_component(_rapidjson_prefix \"\${CMAKE_CURRENT_LIST_DIR}/../../..\" ABSOLUTE)\nset(RapidJSON_FOUND TRUE)\nset(RapidJSON_INCLUDE_DIRS \"\${_rapidjson_prefix}/include\")\nunset(_rapidjson_prefix)") + if(YDB_SDK_INSTALL) + install(DIRECTORY "${RapidJSON_SOURCE_DIR}/include/rapidjson" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT libydb-cpp) + install(FILES "${RapidJSON_DIR}/RapidJSONConfig.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/RapidJSON COMPONENT libydb-cpp) + endif() + endif() + + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "${_ydb_sdk_saved_install_component}") +endif() + +# Ubuntu packages do not provide the pinned base64 and jwt-cpp interfaces. +set(BASE64_WERROR OFF CACHE BOOL "" FORCE) +set(BASE64_BUILD_CLI OFF CACHE BOOL "" FORCE) +CPMAddPackage(NAME base64 GITHUB_REPOSITORY aklomp/base64 + GIT_TAG v${YDB_SDK_BASE64_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS "BASE64_WERROR OFF" "BASE64_BUILD_CLI OFF") +_ydb_sdk_alias_library(aklomp::base64 base64) +_ydb_sdk_apply_aklomp_base64_simd_file_flags("${base64_SOURCE_DIR}") +CPMAddPackage(NAME jwt-cpp GITHUB_REPOSITORY Thalhammer/jwt-cpp + GIT_TAG v${YDB_SDK_JWT_CPP_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS "JWT_BUILD_EXAMPLES OFF" "JWT_BUILD_TESTS OFF") + +if(YDB_SDK_TESTS) + CPMAddPackage( + NAME googletest + GITHUB_REPOSITORY google/googletest + GIT_TAG v${YDB_SDK_GOOGLETEST_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS "INSTALL_GTEST ${YDB_SDK_INSTALL}" "gtest_force_shared_crt ON" + ) +endif() + +if(YDB_SDK_TESTS OR YDB_CPP_SDK_SLO_USE_INSTALLED_SDK) + ydb_sdk_fetch_hdr_histogram() +endif() + +if(YDB_SDK_ENABLE_OTEL_METRICS OR YDB_SDK_ENABLE_OTEL_TRACE OR YDB_SDK_TESTS) + set(_ydb_sdk_otel_saved_install_component "${CMAKE_INSTALL_DEFAULT_COMPONENT_NAME}") + if(YDB_SDK_INSTALL) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp-otel-metrics) + endif() + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(WITH_OTLP_HTTP ON CACHE BOOL "" FORCE) + set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE) + set(WITH_PROMETHEUS OFF CACHE BOOL "" FORCE) + set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) + set(WITH_FUNC_TESTS OFF CACHE BOOL "" FORCE) + set(WITH_BENCHMARK OFF CACHE BOOL "" FORCE) + set(WITH_STL CXX20 CACHE STRING "" FORCE) + set(WITH_ABI_VERSION_1 OFF CACHE BOOL "" FORCE) + set(WITH_ABI_VERSION_2 ON CACHE BOOL "" FORCE) + set(OPENTELEMETRY_INSTALL ${YDB_SDK_INSTALL} CACHE BOOL "" FORCE) + + string(REPLACE "." "_" _ydb_sdk_curl_tag "${YDB_SDK_CURL_VERSION}") + CPMAddPackage( + NAME curl + GITHUB_REPOSITORY curl/curl + GIT_TAG curl-${_ydb_sdk_curl_tag} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + OPTIONS + "BUILD_CURL_EXE OFF" + "BUILD_LIBCURL_DOCS OFF" + "BUILD_MISC_DOCS OFF" + "CURL_DISABLE_INSTALL ${_YDB_SDK_CPM_DISABLE_INSTALL}" + "CURL_USE_LIBPSL OFF" + "CURL_USE_LIBSSH2 OFF" + "CURL_ZLIB OFF" + "CURL_BROTLI OFF" + "CURL_ZSTD OFF" + "HTTP_ONLY ON" + ) + _ydb_sdk_cpm_package_stub(CURL + "set(CURL_FOUND TRUE)\nset(CURL_VERSION ${YDB_SDK_CURL_VERSION})\nset(CURL_VERSION_STRING ${YDB_SDK_CURL_VERSION})") + + CPMAddPackage( + NAME opentelemetry-proto + GITHUB_REPOSITORY open-telemetry/opentelemetry-proto + GIT_TAG v${YDB_SDK_OTEL_PROTO_VERSION} + DOWNLOAD_ONLY YES + ) + set(OTELCPP_PROTO_PATH "${opentelemetry-proto_SOURCE_DIR}" CACHE PATH "" FORCE) + + CPMAddPackage( + NAME opentelemetry-cpp + GITHUB_REPOSITORY open-telemetry/opentelemetry-cpp + GIT_TAG v${YDB_SDK_OPENTELEMETRY_VERSION} + EXCLUDE_FROM_ALL ${_YDB_SDK_CPM_EXCLUDE_FROM_ALL} + ) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "${_ydb_sdk_otel_saved_install_component}") +endif() + +if(NOT YDB_SDK_DEPENDENCY_MODE STREQUAL "SYSTEM") + ydb_sdk_fetch_googleapis() +endif() + +CPMAddPackage( + NAME FastLZSource + GITHUB_REPOSITORY ariya/FastLZ + GIT_TAG ${YDB_SDK_FASTLZ_GIT_TAG} + DOWNLOAD_ONLY YES +) +set(YDB_SDK_FASTLZ_SOURCE_DIR "${FastLZSource_SOURCE_DIR}") + +# The SDK install archives link these separately installed packages; package +# collection must not compile their sources into the SDK a second time. +foreach(_ydb_sdk_cpm_package IN ITEMS + ZLIB abseil-cpp Protobuf c-ares re2 gRPC Brotli xxHash zstd lz4 Snappy + double-conversion RapidJSON base64 jwt-cpp curl opentelemetry-cpp) + if(DEFINED ${_ydb_sdk_cpm_package}_SOURCE_DIR) + set_property(GLOBAL APPEND PROPERTY YDB_SDK_CPM_SOURCE_DIRS + "${${_ydb_sdk_cpm_package}_SOURCE_DIR}") + endif() +endforeach() diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 469df6000ca..1e6120bc3cd 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -1,275 +1,94 @@ +# Platform prerequisites deliberately remain outside CPM. find_package(IDN REQUIRED) find_package(Iconv REQUIRED) find_package(OpenSSL REQUIRED) -find_package(Protobuf CONFIG QUIET) -if (NOT Protobuf_FOUND) - find_package(Protobuf MODULE REQUIRED) -endif() -find_package(gRPC 1.41.0 REQUIRED) -find_package(ZLIB REQUIRED) -find_package(xxHash REQUIRED) -find_package(ZSTD REQUIRED) -find_package(BZip2 REQUIRED) -find_package(LZ4 REQUIRED) -find_package(Snappy 1.1.8 REQUIRED) -set(_ydb_sdk_vendor_base64 "${YDB_SDK_SOURCE_DIR}/third_party/aklomp-base64") -set(_ydb_sdk_vendor_jwt_cpp "${YDB_SDK_SOURCE_DIR}/third_party/jwt-cpp") +find_package(Threads REQUIRED) + +include("${CMAKE_CURRENT_LIST_DIR}/dependencies.cmake") -find_package(base64 QUIET) -if (NOT base64_FOUND) - if (EXISTS "${_ydb_sdk_vendor_base64}/CMakeLists.txt") - set(BASE64_WERROR OFF CACHE BOOL "" FORCE) - set(BASE64_BUILD_CLI OFF CACHE BOOL "" FORCE) - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) - add_subdirectory("${_ydb_sdk_vendor_base64}" "${YDB_SDK_BINARY_DIR}/third_party/aklomp-base64") - _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_vendor_base64}") - if (TARGET base64 AND NOT TARGET aklomp::base64) - add_library(aklomp::base64 ALIAS base64) - endif() - elseif (FETCHCONTENT_FULLY_DISCONNECTED) - message(FATAL_ERROR - "base64 is required but was not found (find_package), vendored sources are missing at " - "'${_ydb_sdk_vendor_base64}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") - else() - include(FetchContent) - FetchContent_Declare( - base64 - GIT_REPOSITORY https://github.com/aklomp/base64.git - GIT_TAG v0.5.2 - ) - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) - FetchContent_MakeAvailable(base64) - get_target_property(_ydb_sdk_b64_src_dir base64 SOURCE_DIR) - _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_b64_src_dir}") - if (TARGET base64 AND NOT TARGET aklomp::base64) - add_library(aklomp::base64 ALIAS base64) - endif() +# OpenTelemetry releases have used both plain and namespaced build-tree target +# names. The SDK consumes one stable spelling in either dependency mode. +if(YDB_SDK_ENABLE_OTEL_METRICS OR YDB_SDK_ENABLE_OTEL_TRACE OR YDB_SDK_TESTS) + if(NOT TARGET opentelemetry-cpp::api AND TARGET opentelemetry_api) + add_library(opentelemetry-cpp::api ALIAS opentelemetry_api) endif() -endif() -find_package(Brotli 1.1.0 REQUIRED) -find_package(jwt-cpp QUIET) -if (NOT jwt-cpp_FOUND) - if (EXISTS "${_ydb_sdk_vendor_jwt_cpp}/CMakeLists.txt") - set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) - add_subdirectory("${_ydb_sdk_vendor_jwt_cpp}" "${YDB_SDK_BINARY_DIR}/third_party/jwt-cpp") - elseif (FETCHCONTENT_FULLY_DISCONNECTED) - message(FATAL_ERROR - "jwt-cpp is required but was not found (find_package), vendored sources are missing at " - "'${_ydb_sdk_vendor_jwt_cpp}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") - else() - include(FetchContent) - FetchContent_Declare( - jwt-cpp - GIT_REPOSITORY https://github.com/Thalhammer/jwt-cpp.git - GIT_TAG v0.6.0 - ) - set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) - FetchContent_MakeAvailable(jwt-cpp) + if(NOT TARGET opentelemetry-cpp::metrics AND TARGET opentelemetry_metrics) + add_library(opentelemetry-cpp::metrics ALIAS opentelemetry_metrics) endif() -endif() -find_package(double-conversion REQUIRED) - -# OpenTelemetry -set(_ydb_sdk_vendor_otel "${YDB_SDK_SOURCE_DIR}/third_party/opentelemetry-cpp") - -if (YDB_SDK_ENABLE_OTEL_METRICS OR YDB_SDK_ENABLE_OTEL_TRACE) - set(BUILD_TESTING OFF CACHE BOOL "" FORCE) - set(WITH_OTLP_HTTP OFF CACHE BOOL "" FORCE) - set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE) - set(WITH_PROMETHEUS OFF CACHE BOOL "" FORCE) - set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) - set(OTELCPP_MAINTAINER_MODE OFF CACHE BOOL "" FORCE) - set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "" FORCE) - set(WITH_STL "CXX20" CACHE STRING "" FORCE) - set(WITH_ABI_VERSION_1 OFF CACHE BOOL "" FORCE) - set(WITH_ABI_VERSION_2 ON CACHE BOOL "" FORCE) - set(OPENTELEMETRY_INSTALL ON CACHE BOOL "" FORCE) - - set(_ydb_sdk_saved_install_component "${CMAKE_INSTALL_DEFAULT_COMPONENT_NAME}") - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp-otel-metrics) - - if (EXISTS "${_ydb_sdk_vendor_otel}/CMakeLists.txt") - add_subdirectory("${_ydb_sdk_vendor_otel}" "${YDB_SDK_BINARY_DIR}/third_party/opentelemetry-cpp") - else() - find_package(opentelemetry-cpp QUIET) - if (NOT opentelemetry-cpp_FOUND) - if (FETCHCONTENT_FULLY_DISCONNECTED) - message(FATAL_ERROR - "opentelemetry-cpp is required but was not found (find_package), vendored sources are missing at " - "'${_ydb_sdk_vendor_otel}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") - else() - include(FetchContent) - FetchContent_Declare( - opentelemetry-cpp - GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git - GIT_TAG v1.26.0 - ) - FetchContent_MakeAvailable(opentelemetry-cpp) - endif() - endif() + if(NOT TARGET opentelemetry-cpp::trace AND TARGET opentelemetry_trace) + add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) endif() - - if (NOT TARGET opentelemetry-cpp::api) - if (TARGET opentelemetry_api) - add_library(opentelemetry-cpp::api ALIAS opentelemetry_api) - endif() - if (TARGET opentelemetry_metrics) - add_library(opentelemetry-cpp::metrics ALIAS opentelemetry_metrics) - endif() - if (TARGET opentelemetry_trace) - add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) - endif() - if (TARGET opentelemetry_common) - add_library(opentelemetry-cpp::common ALIAS opentelemetry_common) - endif() - if (TARGET opentelemetry_resources) - add_library(opentelemetry-cpp::resources ALIAS opentelemetry_resources) - endif() - if (TARGET opentelemetry_version) - add_library(opentelemetry-cpp::version ALIAS opentelemetry_version) - endif() + if(NOT TARGET opentelemetry-cpp::common AND TARGET opentelemetry_common) + add_library(opentelemetry-cpp::common ALIAS opentelemetry_common) endif() - - if (YDB_SDK_INSTALL) - set(_ydb_sdk_otel_binary_dir "${YDB_SDK_BINARY_DIR}/third_party/opentelemetry-cpp") - set(_ydb_sdk_otel_install_component libydb-cpp-otel-metrics) - - if (EXISTS "${_ydb_sdk_otel_binary_dir}/cmake/opentelemetry-cpp") - install(DIRECTORY "${_ydb_sdk_otel_binary_dir}/cmake/opentelemetry-cpp/" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/opentelemetry-cpp" - COMPONENT ${_ydb_sdk_otel_install_component} - USE_SOURCE_PERMISSIONS) - endif() - - if (EXISTS "${_ydb_sdk_vendor_otel}/cmake/find-package-support-functions.cmake") - install(FILES "${_ydb_sdk_vendor_otel}/cmake/find-package-support-functions.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/opentelemetry-cpp" - COMPONENT ${_ydb_sdk_otel_install_component}) - endif() - - foreach(_ydb_sdk_otel_cmake_file IN ITEMS - component-definitions.cmake - thirdparty-dependency-definitions.cmake) - if (EXISTS "${_ydb_sdk_otel_binary_dir}/cmake/${_ydb_sdk_otel_cmake_file}") - install(FILES "${_ydb_sdk_otel_binary_dir}/cmake/${_ydb_sdk_otel_cmake_file}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/opentelemetry-cpp" - COMPONENT ${_ydb_sdk_otel_install_component}) - endif() - endforeach() - - # Export *-target.cmake files are generated during the OTel build; glob at install time. - install(CODE " - file(GLOB _ydb_sdk_otel_export_dirs \"${_ydb_sdk_otel_binary_dir}/CMakeFiles/Export/*\") - foreach(_ydb_sdk_otel_export_dir IN LISTS _ydb_sdk_otel_export_dirs) - if (IS_DIRECTORY \"\${_ydb_sdk_otel_export_dir}\") - file(GLOB _ydb_sdk_otel_export_cmake \"\${_ydb_sdk_otel_export_dir}/*.cmake\") - if (_ydb_sdk_otel_export_cmake) - file(INSTALL \${_ydb_sdk_otel_export_cmake} - DESTINATION \"\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cmake/opentelemetry-cpp\") - endif() - endif() - endforeach() - " COMPONENT ${_ydb_sdk_otel_install_component}) - - foreach(_ydb_sdk_otel_include_dir IN ITEMS api sdk exporters) - if (EXISTS "${_ydb_sdk_vendor_otel}/${_ydb_sdk_otel_include_dir}/include") - install(DIRECTORY "${_ydb_sdk_vendor_otel}/${_ydb_sdk_otel_include_dir}/include/" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" - COMPONENT ${_ydb_sdk_otel_install_component} - USE_SOURCE_PERMISSIONS) - endif() - endforeach() - - install(CODE " - file(GLOB_RECURSE _ydb_sdk_otel_static_libs - \"${_ydb_sdk_otel_binary_dir}/libopentelemetry_*.a\") - if (_ydb_sdk_otel_static_libs) - file(INSTALL \${_ydb_sdk_otel_static_libs} - DESTINATION \"\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\") - endif() - " COMPONENT ${_ydb_sdk_otel_install_component}) + if(NOT TARGET opentelemetry-cpp::resources AND TARGET opentelemetry_resources) + add_library(opentelemetry-cpp::resources ALIAS opentelemetry_resources) + endif() + if(NOT TARGET opentelemetry-cpp::version AND TARGET opentelemetry_version) + add_library(opentelemetry-cpp::version ALIAS opentelemetry_version) endif() - - set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "${_ydb_sdk_saved_install_component}") -endif() - -# RapidJSON -if (YDB_SDK_USE_RAPID_JSON) - find_package(RapidJSON REQUIRED) - - add_library(RapidJSON::RapidJSON INTERFACE IMPORTED) - - target_include_directories(RapidJSON::RapidJSON INTERFACE - ${RAPIDJSON_INCLUDE_DIRS} - ) endif() -# api-common-protos -option(YDB_SDK_USE_SYSTEM_GOOGLEAPIS "Use system-provided yandex-googleapis-api-common-protos" OFF) - -if (YDB_SDK_USE_SYSTEM_GOOGLEAPIS) - find_package(yandex-googleapis-api-common-protos REQUIRED) - add_library(api-common-protos ALIAS yandex-googleapis-api-common-protos::api-common-protos) -elseif (YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET) +# Google API common protos remain independently packageable for Ubuntu, while +# CPM builds generate the same library directly from the pinned source tree. +if(YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET) add_library(api-common-protos ALIAS ${YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET}) +elseif(YDB_SDK_DEPENDENCY_MODE STREQUAL "SYSTEM") + find_package(yandex-googleapis-api-common-protos REQUIRED) + add_library(api-common-protos ALIAS + yandex-googleapis-api-common-protos::api-common-protos) else() - file(MAKE_DIRECTORY ${YDB_SDK_BINARY_DIR}/third_party/api-common-protos) - file(GLOB_RECURSE API_COMMON_PROTOS_SOURCES - ${YDB_SDK_SOURCE_DIR}/third_party/api-common-protos/google/*.proto - ) - - _ydb_sdk_init_proto_library_impl(api-common-protos Off) - - set_property(TARGET api-common-protos PROPERTY - PROTO_NAMESPACE third_party/api-common-protos - ) - - set_property(TARGET api-common-protos APPEND PROPERTY - PROTO_ADDINCL - ./third_party/api-common-protos - ${YDB_SDK_SOURCE_DIR} - ) + file(GLOB_RECURSE API_COMMON_PROTOS_SOURCES CONFIGURE_DEPENDS + "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}/google/*.proto") + _ydb_sdk_add_library(api-common-protos) + target_link_libraries(api-common-protos PUBLIC protobuf::libprotobuf) target_include_directories(api-common-protos PUBLIC $ - ) - - _ydb_sdk_gen_proto_messages(api-common-protos PRIVATE Off ${API_COMMON_PROTOS_SOURCES}) - - _ydb_sdk_install_targets(TARGETS api-common-protos) + $) + + foreach(_proto IN LISTS API_COMMON_PROTOS_SOURCES) + file(RELATIVE_PATH _proto_relative "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}" "${_proto}") + string(REGEX REPLACE "\\.proto$" ".pb.cc" _proto_cc "${_proto_relative}") + string(REGEX REPLACE "\\.proto$" ".pb.h" _proto_h "${_proto_relative}") + set(_proto_cc "${YDB_SDK_BINARY_DIR}/third_party/api-common-protos/${_proto_cc}") + set(_proto_h "${YDB_SDK_BINARY_DIR}/third_party/api-common-protos/${_proto_h}") + get_filename_component(_proto_output_dir "${_proto_cc}" DIRECTORY) + file(MAKE_DIRECTORY "${_proto_output_dir}") + add_custom_command( + OUTPUT "${_proto_cc}" "${_proto_h}" + COMMAND protobuf::protoc + "--cpp_out=${YDB_SDK_BINARY_DIR}/third_party/api-common-protos" + "-I${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}" + "-I${YDB_SDK_PROTOBUF_SOURCE_DIR}/src" + "${_proto_relative}" + WORKING_DIRECTORY "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}" + DEPENDS "${_proto}" + VERBATIM) + target_sources(api-common-protos PRIVATE "${_proto_cc}" "${_proto_h}") + endforeach() + + if(YDB_SDK_INSTALL) + install(DIRECTORY "${YDB_SDK_BINARY_DIR}/third_party/api-common-protos/google" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT libydb-cpp + FILES_MATCHING PATTERN "*.pb.h") + endif() endif() -# FastLZ -add_library(FastLZ - ${YDB_SDK_SOURCE_DIR}/third_party/FastLZ/fastlz.c -) - +add_library(FastLZ "${YDB_SDK_FASTLZ_SOURCE_DIR}/fastlz.c") target_include_directories(FastLZ PUBLIC - $ -) + $) -_ydb_sdk_install_targets(TARGETS FastLZ) - -# nayuki_md5 +# nayuki_md5 is maintained in-tree and is not an external dependency. add_library(nayuki_md5) - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") -target_sources(nayuki_md5 PRIVATE - ${YDB_SDK_SOURCE_DIR}/third_party/nayuki_md5/nayuki_md5-fast-x8664.S -) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + target_sources(nayuki_md5 PRIVATE + ${YDB_SDK_SOURCE_DIR}/third_party/nayuki_md5/nayuki_md5-fast-x8664.S) else() -target_sources(nayuki_md5 PRIVATE - ${YDB_SDK_SOURCE_DIR}/third_party/nayuki_md5/nayuki_md5.c -) + target_sources(nayuki_md5 PRIVATE + ${YDB_SDK_SOURCE_DIR}/third_party/nayuki_md5/nayuki_md5.c) endif() - target_include_directories(nayuki_md5 PUBLIC $ - $ -) - -_ydb_sdk_install_targets(TARGETS nayuki_md5) + $) diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake index 71d43d3ce8b..07212210fc8 100644 --- a/cmake/protobuf.cmake +++ b/cmake/protobuf.cmake @@ -56,11 +56,24 @@ function(_ydb_sdk_init_proto_library_impl Tgt USE_API_COMMON_PROTOS) set(proto_incls ${YDB_SDK_SOURCE_DIR}) + if(YDB_SDK_DEPENDENCY_MODE STREQUAL "CPM") + list(APPEND proto_incls ${YDB_SDK_PROTOBUF_SOURCE_DIR}/src) + else() + find_path(YDB_SDK_PROTOBUF_PROTO_INCLUDE_DIR google/protobuf/descriptor.proto + HINTS ${Protobuf_INCLUDE_DIRS} REQUIRED) + list(APPEND proto_incls ${YDB_SDK_PROTOBUF_PROTO_INCLUDE_DIR}) + endif() + if (USE_API_COMMON_PROTOS) target_link_libraries(${Tgt} PUBLIC api-common-protos ) - list(APPEND proto_incls ${YDB_SDK_SOURCE_DIR}/third_party/api-common-protos) + if(YDB_SDK_DEPENDENCY_MODE STREQUAL "CPM") + list(APPEND proto_incls ${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}) + else() + find_path(YDB_SDK_GOOGLE_PROTO_INCLUDE_DIR google/api/annotations.proto REQUIRED) + list(APPEND proto_incls ${YDB_SDK_GOOGLE_PROTO_INCLUDE_DIR}) + endif() endif() set_property(TARGET ${Tgt} PROPERTY diff --git a/cmake/testing.cmake b/cmake/testing.cmake index 999e6a596d6..ae5a96d516b 100644 --- a/cmake/testing.cmake +++ b/cmake/testing.cmake @@ -1,36 +1,7 @@ enable_testing() -include(FetchContent) - -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG 1.15.2 - FIND_PACKAGE_ARGS NAMES GTest -) - -FetchContent_MakeAvailable(googletest) - include(GoogleTest) -# Set opentelemetry-cpp options as CACHE variables so they're visible to FetchContent -set(WITH_OTLP_HTTP ON CACHE BOOL "" FORCE) -set(WITH_STL CXX20 CACHE STRING "" FORCE) -set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) -set(WITH_FUNC_TESTS OFF CACHE BOOL "" FORCE) -set(WITH_ABI_VERSION_1 OFF CACHE BOOL "" FORCE) -set(WITH_ABI_VERSION_2 ON CACHE BOOL "" FORCE) -set(WITH_BENCHMARK OFF CACHE BOOL "" FORCE) -set(BUILD_TESTING OFF CACHE BOOL "" FORCE) - -FetchContent_Declare( - opentelemetry-cpp - GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git - GIT_TAG v1.26.0 -) - -FetchContent_MakeAvailable(opentelemetry-cpp) - function(add_ydb_test) set(opts GTEST) set(oneval_args NAME WORKING_DIRECTORY OUTPUT_DIRECTORY) @@ -71,7 +42,6 @@ function(add_ydb_test) ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(${YDB_TEST_NAME} PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/cmake/tstring_abi.cpp b/cmake/tstring_abi.cpp new file mode 100644 index 00000000000..7ea714ff0e8 --- /dev/null +++ b/cmake/tstring_abi.cpp @@ -0,0 +1,3 @@ +extern "C" { +extern const int TStringUseCow = 1; +} diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index c6817d87998..6743fad10a8 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -59,10 +59,9 @@ find_package(Snappy 1.1.8 REQUIRED) find_package(base64 REQUIRED) find_package(Brotli 1.1.0 REQUIRED) find_package(jwt-cpp REQUIRED) -find_package(GTest REQUIRED) find_package(double-conversion REQUIRED) -if (@YDB_SDK_USE_SYSTEM_GOOGLEAPIS@) +if ("@YDB_SDK_DEPENDENCY_MODE@" STREQUAL "SYSTEM") find_package(yandex-googleapis-api-common-protos REQUIRED) endif() @@ -78,7 +77,8 @@ if (@YDB_SDK_USE_RAPID_JSON@) endif() endif() -if (@YDB_SDK_ENABLE_OTEL_METRICS@ OR @YDB_SDK_ENABLE_OTEL_TRACE@) +if ("@YDB_SDK_DEPENDENCY_MODE@" STREQUAL "CPM" AND + (@YDB_SDK_ENABLE_OTEL_METRICS@ OR @YDB_SDK_ENABLE_OTEL_TRACE@)) if (_ydb_sdk_needs_otel_metrics OR _ydb_sdk_needs_otel_tracing) get_filename_component(_ydb_sdk_cmake_lib_dir "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE) list(APPEND CMAKE_PREFIX_PATH "${_ydb_sdk_cmake_lib_dir}") diff --git a/examples/basic_example/CMakeLists.txt b/examples/basic_example/CMakeLists.txt index 75d2d1b2538..111227c9ac6 100644 --- a/examples/basic_example/CMakeLists.txt +++ b/examples/basic_example/CMakeLists.txt @@ -31,7 +31,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(basic_example PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/bulk_upsert_simple/CMakeLists.txt b/examples/bulk_upsert_simple/CMakeLists.txt index 4f7c3eca7f9..6e2d435a7c6 100644 --- a/examples/bulk_upsert_simple/CMakeLists.txt +++ b/examples/bulk_upsert_simple/CMakeLists.txt @@ -27,7 +27,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(bulk_upsert_simple PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/otel_tracing/CMakeLists.txt b/examples/otel_tracing/CMakeLists.txt index 0eaf312621b..2c320f63f11 100644 --- a/examples/otel_tracing/CMakeLists.txt +++ b/examples/otel_tracing/CMakeLists.txt @@ -34,8 +34,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(otel_tracing_example PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) -endif() \ No newline at end of file +endif() diff --git a/examples/pagination/CMakeLists.txt b/examples/pagination/CMakeLists.txt index 0936f385585..22f30d26df7 100644 --- a/examples/pagination/CMakeLists.txt +++ b/examples/pagination/CMakeLists.txt @@ -29,8 +29,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(pagination PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) -endif() \ No newline at end of file +endif() diff --git a/examples/secondary_index/CMakeLists.txt b/examples/secondary_index/CMakeLists.txt index 6030de5f7f0..be407f40627 100644 --- a/examples/secondary_index/CMakeLists.txt +++ b/examples/secondary_index/CMakeLists.txt @@ -34,7 +34,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(secondary_index PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/secondary_index_builtin/CMakeLists.txt b/examples/secondary_index_builtin/CMakeLists.txt index b46cc79159c..fda8b109c24 100644 --- a/examples/secondary_index_builtin/CMakeLists.txt +++ b/examples/secondary_index_builtin/CMakeLists.txt @@ -33,7 +33,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(secondary_index_builtin PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/time/CMakeLists.txt b/examples/time/CMakeLists.txt index 3019fd04851..31a898e0489 100644 --- a/examples/time/CMakeLists.txt +++ b/examples/time/CMakeLists.txt @@ -30,7 +30,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(time_example PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/topic_reader/eventloop/CMakeLists.txt b/examples/topic_reader/eventloop/CMakeLists.txt index 2cdc984955f..9919d0d2a41 100644 --- a/examples/topic_reader/eventloop/CMakeLists.txt +++ b/examples/topic_reader/eventloop/CMakeLists.txt @@ -27,7 +27,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(persqueue_reader_eventloop PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/topic_reader/simple/CMakeLists.txt b/examples/topic_reader/simple/CMakeLists.txt index 68846ab215c..d002247f814 100644 --- a/examples/topic_reader/simple/CMakeLists.txt +++ b/examples/topic_reader/simple/CMakeLists.txt @@ -27,7 +27,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(simple_persqueue_reader PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/topic_reader/transaction/CMakeLists.txt b/examples/topic_reader/transaction/CMakeLists.txt index 64d30b4d8c6..bc4abc6017f 100644 --- a/examples/topic_reader/transaction/CMakeLists.txt +++ b/examples/topic_reader/transaction/CMakeLists.txt @@ -29,7 +29,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(read_from_topic_in_transaction PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/topic_writer/producer/basic_write/CMakeLists.txt b/examples/topic_writer/producer/basic_write/CMakeLists.txt index f95bbedad67..9707febdd32 100644 --- a/examples/topic_writer/producer/basic_write/CMakeLists.txt +++ b/examples/topic_writer/producer/basic_write/CMakeLists.txt @@ -29,7 +29,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(topic_writer_producer_buffer_overloaded PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/topic_writer/transaction/CMakeLists.txt b/examples/topic_writer/transaction/CMakeLists.txt index cbb717958c8..3d3fe2c02ea 100644 --- a/examples/topic_writer/transaction/CMakeLists.txt +++ b/examples/topic_writer/transaction/CMakeLists.txt @@ -29,7 +29,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(topic_writer_transaction PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/ttl/CMakeLists.txt b/examples/ttl/CMakeLists.txt index 48a004b4cc6..ba0d57bc51b 100644 --- a/examples/ttl/CMakeLists.txt +++ b/examples/ttl/CMakeLists.txt @@ -28,7 +28,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(ttl PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/vector_index/CMakeLists.txt b/examples/vector_index/CMakeLists.txt index 19249951e37..abf8adededd 100644 --- a/examples/vector_index/CMakeLists.txt +++ b/examples/vector_index/CMakeLists.txt @@ -29,7 +29,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(vector_index PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/examples/vector_index_builtin/CMakeLists.txt b/examples/vector_index_builtin/CMakeLists.txt index 75c97b2c3c2..d6932523106 100644 --- a/examples/vector_index_builtin/CMakeLists.txt +++ b/examples/vector_index_builtin/CMakeLists.txt @@ -32,7 +32,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(vector_index_builtin PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/library/cpp/charset/CMakeLists.txt b/library/cpp/charset/CMakeLists.txt index 1825f70e8b6..e63d652a501 100644 --- a/library/cpp/charset/CMakeLists.txt +++ b/library/cpp/charset/CMakeLists.txt @@ -19,7 +19,7 @@ if (YDB_SDK_TESTS) recyr_int_ut.cpp wide_ut.cpp LINK_LIBRARIES - charset + ydb-sdk-charset cpp-testing-unittest_main LABELS unit @@ -42,18 +42,19 @@ target_sources(charset-lite _ydb_sdk_install_targets(TARGETS charset-lite) -_ydb_sdk_add_library(charset) -target_link_libraries(charset +_ydb_sdk_add_library(ydb-sdk-charset) +set_target_properties(ydb-sdk-charset PROPERTIES EXPORT_NAME charset) +target_link_libraries(ydb-sdk-charset PUBLIC yutil charset-lite PRIVATE Iconv::Iconv ) -target_sources(charset +target_sources(ydb-sdk-charset PRIVATE decodeunknownplane.cpp wide.cpp iconv.cpp ) -_ydb_sdk_install_targets(TARGETS charset) +_ydb_sdk_install_targets(TARGETS ydb-sdk-charset) diff --git a/library/cpp/http/fetch/CMakeLists.txt b/library/cpp/http/fetch/CMakeLists.txt index 821b12ed803..3fe0d0bfcb5 100644 --- a/library/cpp/http/fetch/CMakeLists.txt +++ b/library/cpp/http/fetch/CMakeLists.txt @@ -31,7 +31,7 @@ target_link_libraries(http-fetch uri ZLIB::ZLIB PRIVATE - charset + ydb-sdk-charset digest-md5 enum_serialization_runtime ) diff --git a/library/cpp/uri/CMakeLists.txt b/library/cpp/uri/CMakeLists.txt index b06e3987e98..c63c39a0b52 100644 --- a/library/cpp/uri/CMakeLists.txt +++ b/library/cpp/uri/CMakeLists.txt @@ -3,7 +3,7 @@ _ydb_sdk_add_library(uri) target_link_libraries(uri PUBLIC yutil - charset + ydb-sdk-charset PRIVATE IDN::IDN ) diff --git a/scripts/build_cpack_deb_packages.sh b/scripts/build_cpack_deb_packages.sh index 91f0b8941e2..f6bd4176f8b 100755 --- a/scripts/build_cpack_deb_packages.sh +++ b/scripts/build_cpack_deb_packages.sh @@ -1,141 +1,73 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail -SOURCE_DIR="${SOURCE_DIR:-/source}" -if [ ! -d "$SOURCE_DIR" ]; then - SOURCE_DIR=$(dirname "$(realpath "$0")")/.. - SOURCE_DIR=$(realpath "$SOURCE_DIR") -fi - -OUTPUT_DIR="${1:-build-deb}" +SOURCE_DIR="${SOURCE_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" +OUTPUT_DIR="${1:-${SOURCE_DIR}/build-deb/packages}" +BUILD_ROOT="${YDB_DEB_BUILD_ROOT:-${SOURCE_DIR}/build-deb}" +CPM_SOURCE_CACHE="${CPM_SOURCE_CACHE:-${SOURCE_DIR}/.cache/cpm}" +CCACHE_DIR="${CCACHE_DIR:-${SOURCE_DIR}/.cache/ccache}" -cd "$SOURCE_DIR" -mkdir -p "$OUTPUT_DIR" -OUTPUT_DIR=$(realpath "$OUTPUT_DIR") +mkdir -p "$OUTPUT_DIR" "$BUILD_ROOT" "$CPM_SOURCE_CACHE" "$CCACHE_DIR" +OUTPUT_DIR="$(realpath "$OUTPUT_DIR")" +BUILD_ROOT="$(realpath "$BUILD_ROOT")" -export DEBIAN_FRONTEND=noninteractive -export CCACHE_DIR="${CCACHE_DIR:-/root/.ccache}" -mkdir -p "$CCACHE_DIR" +if [ "${EUID}" -eq 0 ]; then + SUDO=() +else + SUDO=(sudo) +fi if [ "${YDB_DEB_INSTALL_DEPS:-1}" = "1" ]; then - apt-get update - apt-get install -y --no-install-recommends \ - build-essential \ - ccache \ - cmake \ - pkg-config \ - git \ - libidn11-dev \ - libssl-dev \ - zlib1g-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libgrpc++-dev \ - protobuf-compiler-grpc \ - libbrotli-dev \ - liblz4-dev \ - libzstd-dev \ - libbz2-dev \ - libxxhash-dev \ - libsnappy-dev \ - libdouble-conversion-dev \ - libgtest-dev \ - libre2-dev \ - libc-ares-dev \ - rapidjson-dev \ - python3 \ - python3-six \ - ragel \ - yasm + export DEBIAN_FRONTEND=noninteractive + "${SUDO[@]}" apt-get update + "${SUDO[@]}" apt-get install -y --no-install-recommends \ + build-essential ca-certificates ccache cmake ninja-build pkg-config git \ + libidn11-dev libssl-dev zlib1g-dev \ + libprotobuf-dev protobuf-compiler libgrpc++-dev protobuf-compiler-grpc \ + libbrotli-dev liblz4-dev libzstd-dev libbz2-dev libxxhash-dev \ + libsnappy-dev libdouble-conversion-dev libre2-dev \ + libc-ares-dev rapidjson-dev python3 ragel yasm fi -touch_existing_sources() { - for path in "$@"; do - if [ -e "$path" ]; then - find "$path" -type f -exec touch {} + - fi - done -} - -fix_output_permissions() { - local uid="${HOST_UID:-$(stat -c '%u' "$SOURCE_DIR" 2>/dev/null || true)}" - local gid="${HOST_GID:-$(stat -c '%g' "$SOURCE_DIR" 2>/dev/null || true)}" - local paths=() - - for path in "$@"; do - [ ! -e "$path" ] || paths+=("$path") - done - if [ "${#paths[@]}" -eq 0 ]; then - return - fi - - if [[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ ]] && chown -R "$uid:$gid" "${paths[@]}" 2>/dev/null; then - return - fi - - chmod -R a+rX "${paths[@]}" 2>/dev/null || true -} - -rm -f build_googleapis_deb/*.deb build-deb/*.deb "$OUTPUT_DIR"/*.deb 2>/dev/null || true - +export CPM_SOURCE_CACHE CCACHE_DIR +compiler_cache_args=() if command -v ccache >/dev/null 2>&1; then - ccache --zero-stats >/dev/null || true - CMAKE_COMPILER_LAUNCHER_ARGS=( + compiler_cache_args=( -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ) -else - CMAKE_COMPILER_LAUNCHER_ARGS=() fi -# Restored build-cache archives can have outputs newer than the checkout. -# Touch package inputs so the incremental build cannot hide source changes. -touch_existing_sources scripts/googleapis_deb third_party/api-common-protos -cmake -S scripts/googleapis_deb -B build_googleapis_deb \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}" -cmake --build build_googleapis_deb -j"$(nproc)" -cmake --build build_googleapis_deb --target package -dpkg -i build_googleapis_deb/*.deb - -./scripts/generate-debian-directory.sh +google_build="${BUILD_ROOT}/googleapis" +sdk_build="${BUILD_ROOT}/sdk" -touch_existing_sources \ - CMakeLists.txt \ - cmake \ - contrib \ - include \ - library \ - plugins \ - scripts/build_cpack_deb_packages.sh \ - scripts/generate-debian-directory.sh \ - src \ - third_party \ - tools \ - util +cmake -S "${SOURCE_DIR}/scripts/googleapis_deb" -B "$google_build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCPM_SOURCE_CACHE="$CPM_SOURCE_CACHE" \ + -DFETCHCONTENT_FULLY_DISCONNECTED="${YDB_DEB_FULLY_DISCONNECTED:-OFF}" \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + "${compiler_cache_args[@]}" +cmake --build "$google_build" --target package --parallel +"${SUDO[@]}" dpkg -i "$google_build"/*.deb -cmake -S . -B build-deb \ +cmake -S "$SOURCE_DIR" -B "$sdk_build" -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DCMAKE_PREFIX_PATH=/usr/share/yandex \ + -DCPM_SOURCE_CACHE="$CPM_SOURCE_CACHE" \ + -DFETCHCONTENT_FULLY_DISCONNECTED="${YDB_DEB_FULLY_DISCONNECTED:-OFF}" \ + -DYDB_SDK_DEPENDENCY_MODE=SYSTEM \ -DYDB_SDK_INSTALL=ON \ -DYDB_SDK_EXAMPLES=OFF \ -DYDB_SDK_TESTS=OFF \ -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ - -DBUILD_SHARED_LIBS=OFF \ - -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/share/yandex" \ - "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}" -cmake --build build-deb --target package -j"$(nproc)" + "${compiler_cache_args[@]}" +cmake --build "$sdk_build" --target package --parallel -cp -f build_googleapis_deb/*.deb "$OUTPUT_DIR"/ -if [ "$(realpath build-deb)" != "$OUTPUT_DIR" ]; then - cp -f build-deb/*.deb "$OUTPUT_DIR"/ -fi +cp -f "$google_build"/*.deb "$sdk_build"/*.deb "$OUTPUT_DIR"/ if command -v ccache >/dev/null 2>&1; then ccache --show-stats || true fi - -fix_output_permissions "$CCACHE_DIR" build_googleapis_deb build-deb "$OUTPUT_DIR" debian ls -la "$OUTPUT_DIR" diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh deleted file mode 100755 index 46bee455b33..00000000000 --- a/scripts/generate-debian-directory.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/bash -set -e - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") - -# --------------------------------------------------------------------------- -# Parse arguments -# --------------------------------------------------------------------------- -SERIES="noble" -while [[ $# -gt 0 ]]; do - case "$1" in - --series) - SERIES="$2" - shift 2 - ;; - --series=*) - SERIES="${1#*=}" - shift - ;; - *) - echo "Unknown option: $1" >&2 - echo "Usage: $0 [--series ]" >&2 - exit 1 - ;; - esac -done - -cd "$SOURCE_DIR" - -VERSION=$(grep -E 'YDB_SDK_VERSION = "[0-9]+\.[0-9]+\.[0-9]+"' src/version.h | sed -E 's/.*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/') -if [ -z "$VERSION" ]; then - echo "Error: Could not extract YDB_SDK_VERSION from src/version.h" - exit 1 -fi - -GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") - -mkdir -p debian/ - -CHANGELOG_FILE="debian/changelog" - -DEB_VERSION="${VERSION}-1~${SERIES}1" - -cat < "$CHANGELOG_FILE" -ydb-cpp-sdk (${DEB_VERSION}) ${SERIES}; urgency=low - - * Automated package build - * Git commit: ${GIT_COMMIT} - - -- YDB Team $(date -R) -EOF_CHANGELOG - -cat < debian/control -Source: ydb-cpp-sdk -Section: libdevel -Priority: optional -Maintainer: YDB Team -Build-Depends: debhelper-compat (= 13), - cmake, - git, - pkg-config, - python3, - python3-six, - ragel, - yasm, - libidn11-dev, - libssl-dev, - zlib1g-dev, - libprotobuf-dev, - protobuf-compiler, - libgrpc++-dev, - protobuf-compiler-grpc, - libbrotli-dev, - liblz4-dev, - libzstd-dev, - libbz2-dev, - libxxhash-dev, - libsnappy-dev, - libdouble-conversion-dev, - libgtest-dev, - libre2-dev, - libc-ares-dev, - rapidjson-dev, - yandex-googleapis-api-common-protos -Standards-Version: 4.6.2 -Homepage: https://ydb.tech -Rules-Requires-Root: no - -Package: libydb-cpp-dev -Architecture: any -Depends: \${misc:Depends}, - libidn11-dev, - libssl-dev, - zlib1g-dev, - libprotobuf-dev, - libgrpc++-dev, - libbrotli-dev, - liblz4-dev, - libzstd-dev, - libbz2-dev, - libxxhash-dev, - libsnappy-dev, - libdouble-conversion-dev, - libgtest-dev, - libre2-dev, - libc-ares-dev, - rapidjson-dev, - yandex-googleapis-api-common-protos -Description: YDB C++ SDK development files - Static library, public headers and CMake package files for YDB C++ SDK. - Bundled third-party static libraries and headers: libbase64 (aklomp-base64), - jwt-cpp (includes picojson). - -Package: libydb-cpp-iam-dev -Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) -Description: YDB C++ SDK IAM plugin development files - Static library for YDB C++ SDK IAM credentials plugin. - -Package: libydb-cpp-otel-metrics-dev -Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) -Description: YDB C++ SDK OpenTelemetry metrics plugin development files - Static library and headers for YDB C++ SDK OpenTelemetry metrics plugin. - Includes vendored opentelemetry-cpp headers, static libraries, CMake and pkg-config files. - -Package: libydb-cpp-otel-tracing-dev -Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}), - libydb-cpp-otel-metrics-dev (= \${binary:Version}) -Description: YDB C++ SDK OpenTelemetry tracing plugin development files - Static library and headers for YDB C++ SDK OpenTelemetry tracing plugin. - Requires libydb-cpp-otel-metrics-dev for vendored opentelemetry-cpp artifacts. -EOF_CONTROL - -cat <<'EOF_RULES' > debian/rules -#!/usr/bin/make -f - -export DEB_BUILD_MAINT_OPTIONS = hardening=+all - -%: - dh $@ --buildsystem=cmake - -override_dh_auto_configure: - dh_auto_configure -- \ - -DCMAKE_BUILD_TYPE=Release \ - -DYDB_SDK_INSTALL=ON \ - -DYDB_SDK_EXAMPLES=OFF \ - -DYDB_SDK_TESTS=OFF \ - -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ - -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ - -DBUILD_SHARED_LIBS=OFF \ - -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/share/yandex" -EOF_RULES -chmod +x debian/rules - -cat < debian/libydb-cpp-dev.install -debian/tmp/usr/share/yandex/lib/*/libydb-cpp.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/include/ydb-cpp-sdk usr/share/yandex/include/ -debian/tmp/usr/share/yandex/include/__ydb_sdk_special_headers usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/*.cmake usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/Modules usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ydb-cpp-sdk-core-targets*.cmake usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ -debian/tmp/usr/share/yandex/include/libbase64.h usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/libbase64.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/base64 usr/share/yandex/lib/*/cmake/ -debian/tmp/usr/share/yandex/include/picojson usr/share/yandex/include/ -debian/tmp/usr/share/yandex/include/jwt-cpp usr/share/yandex/include/ -debian/tmp/usr/share/yandex/cmake/jwt-cpp* usr/share/yandex/cmake/ -EOF_INSTALL - -cat < debian/libydb-cpp-iam-dev.install -debian/tmp/usr/share/yandex/lib/*/libydb-cpp-iam.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ydb-cpp-sdk-iam-targets*.cmake usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ -EOF_INSTALL - -cat < debian/libydb-cpp-otel-metrics-dev.install -debian/tmp/usr/share/yandex/lib/*/libydb-cpp-otel-metrics.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ydb-cpp-sdk-otel-metrics-targets*.cmake usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ -debian/tmp/usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/metrics.h usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/ -debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/libopentelemetry_* usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/*/cmake/ -debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/*/pkgconfig/ -EOF_INSTALL - -cat < debian/libydb-cpp-otel-tracing-dev.install -debian/tmp/usr/share/yandex/lib/*/libydb-cpp-otel-tracing.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ydb-cpp-sdk-otel-tracing-targets*.cmake usr/share/yandex/lib/*/cmake/ydb-cpp-sdk/release/ -debian/tmp/usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/trace.h usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/ -EOF_INSTALL - - -mkdir -p debian/source -echo "3.0 (quilt)" > debian/source/format - -echo "Generated debian directory for version ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/generate_vcs_info.py b/scripts/generate_vcs_info.py index b311be5617d..f158d869c16 100644 --- a/scripts/generate_vcs_info.py +++ b/scripts/generate_vcs_info.py @@ -6,7 +6,6 @@ import subprocess import sys import time -import six as six_ INDENT = " " * 4 @@ -21,10 +20,7 @@ def _get_vcs_dictionary(vcs_type, *arg): def _get_user_locale(): try: - if six_.PY3: - return [locale.getencoding()] - else: - return [locale.getdefaultlocale()[1]] + return [locale.getpreferredencoding(False)] except Exception: return [] @@ -99,8 +95,6 @@ def external_data(arc_root): # using local 'Popen' wrapper commit = _SystemInfo._system_command_call(['git'] + hash_args, env=env, cwd=arc_root).rstrip() author = _SystemInfo._system_command_call(['git'] + author_args, env=env, cwd=arc_root) - commit = _SystemInfo._system_command_call(['git'] + hash_args, env=env, cwd=arc_root).rstrip() - author = _SystemInfo._system_command_call(['git'] + author_args, env=env, cwd=arc_root) summary = _SystemInfo._system_command_call(['git'] + summary_args, env=env, cwd=arc_root) svn_id = _SystemInfo._system_command_call(['git'] + svn_args, env=env, cwd=arc_root) if not svn_id: @@ -116,7 +110,7 @@ def external_data(arc_root): except Exception: branch_info = [''.encode('utf-8')] - depth = six_.text_type(_GitVersion._get_git_depth(env, arc_root)).encode('utf-8') + depth = str(_GitVersion._get_git_depth(env, arc_root)).encode('utf-8') # logger.debug('Git info commit:{}, author:{}, summary:{}, svn_id:{}'.format(commit, author, summary, svn_id)) return [commit, author, summary, svn_id, tag_info[0], branch_info[0], depth] @@ -175,7 +169,7 @@ def get_locale(cls): @staticmethod def _to_text(s): - if isinstance(s, six_.binary_type): + if isinstance(s, bytes): return s.decode(_SystemInfo.get_locale(), errors='replace') return s @@ -239,7 +233,7 @@ def _system_command_call(command, **kwargs): try: import ctypes - msg = six_.text_type(ctypes.FormatError(e.winerror), _SystemInfo.get_locale()).encode('utf-8') + msg = ctypes.FormatError(e.winerror).encode('utf-8') except ImportError: pass # logger.debug('System command call {} failed [{}]: {}\n'.format(command, errcodes, msg)) diff --git a/scripts/googleapis_deb/CMakeLists.txt b/scripts/googleapis_deb/CMakeLists.txt index 0c96c2358d4..a2587f7538f 100644 --- a/scripts/googleapis_deb/CMakeLists.txt +++ b/scripts/googleapis_deb/CMakeLists.txt @@ -1,18 +1,13 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.22) project(yandex-googleapis-api-common-protos) set(CMAKE_CXX_STANDARD 20) find_package(Protobuf REQUIRED) -set(PROTOS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/api-common-protos") -get_filename_component(PROTOS_DIR "${PROTOS_DIR}" ABSOLUTE) - -if(NOT EXISTS "${PROTOS_DIR}/google") - message(FATAL_ERROR - "api-common-protos sources not found under \"${PROTOS_DIR}/google\". " - "Initialize git submodules / third_party checkout.") -endif() +set(YDB_SDK_DEPENDENCIES_GOOGLEAPIS_ONLY ON) +include("${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/dependencies.cmake") +set(PROTOS_DIR "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}") file(GLOB_RECURSE PROTO_FILES "${PROTOS_DIR}/google/*.proto") @@ -69,6 +64,11 @@ install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/google" FILES_MATCHING PATTERN "*.pb.h" ) +install(DIRECTORY "${YDB_SDK_GOOGLEAPIS_SOURCE_DIR}/google" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.proto" +) + install(EXPORT yandex-googleapis-api-common-protos-targets FILE yandex-googleapis-api-common-protosTargets.cmake NAMESPACE yandex-googleapis-api-common-protos:: @@ -90,6 +90,7 @@ install(FILES set(CPACK_PACKAGE_NAME "yandex-googleapis-api-common-protos") set(CPACK_PACKAGE_VERSION "1.0.0") set(CPACK_GENERATOR "DEB") +set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_DEBIAN_PACKAGE_MAINTAINER "YDB Team ") set(CPACK_DEBIAN_PACKAGE_DEPENDS "libprotobuf-dev") set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/share/yandex") diff --git a/scripts/test_copy_sources_plugins.sh b/scripts/test_copy_sources_plugins.sh index 010e18de7cc..af7c70ec5f3 100755 --- a/scripts/test_copy_sources_plugins.sh +++ b/scripts/test_copy_sources_plugins.sh @@ -77,7 +77,7 @@ echo "# OSS_CMAKE" > "$OSS/plugins/metrics/CMakeLists.txt" echo "# OSS_CMAKE" > "$OSS/plugins/trace/otel/CMakeLists.txt" echo "# OSS_CMAKE" > "$OSS/plugins/metrics/otel/CMakeLists.txt" -touch "$OSS/.gitignore" "$OSS/.gitmodules" "$OSS/CMakePresets.json" "$OSS/CMakeLists.txt" +touch "$OSS/.gitignore" "$OSS/CMakePresets.json" "$OSS/CMakeLists.txt" touch "$OSS/LICENSE" "$OSS/README.md" touch "$OSS/tests/slo_workloads/.dockerignore" "$OSS/tests/slo_workloads/Dockerfile" echo "OSS_SLO_WORKLOAD" > "$OSS/tests/slo_workloads/key_value/main.cpp" diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index 194c6090d8f..dc5bf25ba45 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash +set -euo pipefail if [ "$#" -ne 1 ]; then echo "Usage: $0 " @@ -8,31 +8,33 @@ fi DEB_DIR=$(realpath "$1") SCRIPT_DIR=$(dirname "$(realpath "$0")") -SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") TEST_DIR=$(realpath "$SCRIPT_DIR/../tests/deb_package") +shopt -s nullglob +for package in \ + yandex-googleapis-api-common-protos \ + libydb-cpp-dev \ + libydb-cpp-iam-dev \ + libydb-cpp-otel-metrics-dev \ + libydb-cpp-otel-tracing-dev; do + files=("$DEB_DIR/${package}_"*.deb) + if [ "${#files[@]}" -ne 1 ]; then + echo "Expected exactly one ${package} package, found ${#files[@]}" + exit 1 + fi +done + echo "Building test Docker image..." docker build -t ydb-cpp-sdk-deb-test "$TEST_DIR" echo "Running test container..." docker run --rm \ -v "$DEB_DIR:/deb_packages:ro" \ - -v "$SOURCE_DIR:/source:ro" \ ydb-cpp-sdk-deb-test \ bash -c ' set -e apt-get update - -if ! compgen -G "/deb_packages/yandex-googleapis-api-common-protos*.deb" > /dev/null; then - echo "Building yandex-googleapis-api-common-protos .deb..." - cmake -S /source/scripts/googleapis_deb -B /tmp/build_googleapis_deb \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex - cmake --build /tmp/build_googleapis_deb -j"$(nproc)" - cmake --build /tmp/build_googleapis_deb --target package - dpkg -i /tmp/build_googleapis_deb/*.deb -else - dpkg -i /deb_packages/yandex-googleapis-api-common-protos*.deb -fi +apt-get install -y /deb_packages/yandex-googleapis-api-common-protos_*.deb configure_components() { local name="$1" diff --git a/scripts/test_dpkg_buildpackage.sh b/scripts/test_dpkg_buildpackage.sh deleted file mode 100755 index 6ca1f22a46d..00000000000 --- a/scripts/test_dpkg_buildpackage.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -e - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") - -echo "Building test Docker image for dpkg-buildpackage..." - -# We need a Dockerfile that has all the build dependencies for dpkg-buildpackage -DOCKERFILE=$(mktemp) -cat < "$DOCKERFILE" -FROM ubuntu:24.04 -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \\ - build-essential \\ - cmake \\ - pkg-config \\ - git \\ - libidn11-dev \\ - libssl-dev \\ - zlib1g-dev \\ - libprotobuf-dev \\ - protobuf-compiler \\ - libgrpc++-dev \\ - protobuf-compiler-grpc \\ - libbrotli-dev \\ - liblz4-dev \\ - libzstd-dev \\ - libbz2-dev \\ - libxxhash-dev \\ - libsnappy-dev \\ - libdouble-conversion-dev \\ - libgtest-dev \\ - libre2-dev \\ - libc-ares-dev \\ - rapidjson-dev \\ - debhelper \\ - dpkg-dev \\ - python3 \\ - python3-six \\ - ragel \\ - yasm \\ - && rm -rf /var/lib/apt/lists/* -WORKDIR /source -EOF - -docker build -t ydb-cpp-sdk-dpkg-test -f "$DOCKERFILE" . -rm -f "$DOCKERFILE" - -# Prepare a clean tarball or just copy the directory inside the container -echo "Running dpkg-buildpackage in the container..." -docker run --rm \ - -v "$SOURCE_DIR:/source:ro" \ - ydb-cpp-sdk-dpkg-test \ - bash -c "cp -r /source /tmp/source && cd /tmp/source && \ - rm -rf build_googleapis_deb && \ - cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex && \ - cmake --build build_googleapis_deb -j\$(nproc) && \ - cmake --build build_googleapis_deb --target package && \ - dpkg -i build_googleapis_deb/*.deb && \ - ./scripts/generate-debian-directory.sh && \ - dpkg-buildpackage -us -uc -b -d -j\$(nproc)" - -echo "Test successful! dpkg-buildpackage works." \ No newline at end of file diff --git a/tests/deb_package/Dockerfile b/tests/deb_package/Dockerfile index 89a02daea37..a067a5c8d5c 100644 --- a/tests/deb_package/Dockerfile +++ b/tests/deb_package/Dockerfile @@ -2,29 +2,10 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ + ca-certificates \ cmake \ - pkg-config \ - git \ - libidn11-dev \ - libssl-dev \ - zlib1g-dev \ - libprotobuf-dev \ - protobuf-compiler \ - libgrpc++-dev \ - protobuf-compiler-grpc \ - libbrotli-dev \ - liblz4-dev \ - libzstd-dev \ - libbz2-dev \ - libxxhash-dev \ - libsnappy-dev \ - libdouble-conversion-dev \ - libgtest-dev \ - libre2-dev \ - libc-ares-dev \ - rapidjson-dev \ && rm -rf /var/lib/apt/lists/* COPY test_project /test_project diff --git a/tests/slo_workloads/Dockerfile b/tests/slo_workloads/Dockerfile index e51001c12cf..d9534f993bf 100644 --- a/tests/slo_workloads/Dockerfile +++ b/tests/slo_workloads/Dockerfile @@ -1,191 +1,49 @@ # syntax=docker/dockerfile:1.7 -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG PRESET=release-test-clang -ARG SDK_PRESET=release-clang +ENV DEBIAN_FRONTEND=noninteractive \ + CPM_SOURCE_CACHE=/root/.cache/cpm \ + CCACHE_DIR=/root/.ccache \ + CCACHE_MAXSIZE=2G \ + YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk -# ccache settings consumed by the configure/build steps below. The cache dir -# is materialised by the BuildKit cache mount on those RUN steps; values -# elsewhere in the image are inert. -ENV CCACHE_DIR=/root/.ccache -ENV CCACHE_MAXSIZE=2G -ENV CCACHE_COMPRESS=true -ENV CCACHE_COMPILERCHECK=content - -# Every RUN that hits the network retries on transient failures so one -# flake doesn't throw away 30 min of previous build work. apt gets five -# Acquire retries + 60 s timeouts; wget gets the equivalent via WGET_OPTS. -RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \ - echo 'Acquire::http::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries && \ - echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries - -ENV WGET_OPTS="--tries=5 --waitretry=15 --timeout=60 --retry-connrefused --retry-on-http-error=500,502,503,504" -ENV YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk -ENV YDB_DEPS_PREFIX_PATH="${YDB_CPP_SDK_INSTALL_PREFIX};/root/ydb_deps/absl;/root/ydb_deps/protobuf;/root/ydb_deps/grpc;/root/ydb_deps/brotli" - -# Install software-properties-common and add the gcc-13 PPA. -# Acquire::Retries only retries HTTP errors; TCP connect timeouts to -# ppa.launchpadcontent.net still drop through and kill the step. Wrap the -# whole command in a shell retry loop with exponential backoff so a CDN -# blip doesn't throw away 30 minutes of downstream build work. -RUN for i in 1 2 3 4 5; do \ - apt-get -y update && \ - apt-get -y install software-properties-common && \ - add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ - apt-get -y update && \ - break; \ - echo "add-apt-repository attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - done && \ - apt-cache show gcc-13 > /dev/null # fail fast if PPA never came up - -# Install C++ tools and libraries -RUN for i in 1 2 3 4 5; do \ - apt-get -y install \ - git gdb wget ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 && \ - break; \ - echo "apt-get install attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - apt-get -y update || true; \ - done && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Install CMake -ENV CMAKE_VERSION=3.27.7 -RUN wget $WGET_OPTS https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ - -q -O cmake-install.sh \ - && chmod u+x cmake-install.sh \ - && ./cmake-install.sh --skip-license --prefix=/usr/local \ - && rm cmake-install.sh - -# Install LLVM -ENV LLVM_VERSION=17 -RUN wget $WGET_OPTS https://apt.llvm.org/llvm.sh && \ - chmod u+x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} && \ - rm llvm.sh - -# Update alternatives to use the selected Clang version by default -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000 - -# Update alternatives to use gcc-13 by default -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - -# Install abseil-cpp -ENV ABSEIL_CPP_VERSION=20230802.0 -ENV ABSEIL_CPP_INSTALL_DIR=/root/ydb_deps/absl -RUN wget $WGET_OPTS -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \ - tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} && \ - rm -rf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz abseil-cpp-${ABSEIL_CPP_VERSION} - -# Install protobuf -ENV PROTOBUF_VERSION=25.0 -ENV PROTOBUF_INSTALL_DIR=/root/ydb_deps/protobuf -RUN wget $WGET_OPTS -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ - tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} && \ - rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} - -# Install grpc -ENV GRPC_VERSION=1.60.2 -ENV GRPC_INSTALL_DIR=/root/ydb_deps/grpc -RUN wget $WGET_OPTS -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ - tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ - -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR} && \ - rm -rf grpc-${GRPC_VERSION}.tar.gz grpc-${GRPC_VERSION} - -# Install base64 -ENV BASE64_VERSION=0.5.2 -ENV BASE64_INSTALL_DIR=/root/ydb_deps/base64 -RUN wget $WGET_OPTS -O base64-${BASE64_VERSION}.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v${BASE64_VERSION}.tar.gz && \ - tar -xvzf base64-${BASE64_VERSION}.tar.gz && cd base64-${BASE64_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${BASE64_INSTALL_DIR} && \ - rm -rf base64-${BASE64_VERSION}.tar.gz base64-${BASE64_VERSION} - -# Install brotli -ENV BROTLI_VERSION=1.1.0 -ENV BROTLI_INSTALL_DIR=/root/ydb_deps/brotli -RUN wget $WGET_OPTS -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ - tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${BROTLI_INSTALL_DIR} .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release && \ - rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} - -# Install jwt-cpp -ENV JWT_CPP_VERSION=0.7.0 -ENV JWT_CPP_INSTALL_DIR=/root/ydb_deps/jwt-cpp -RUN wget $WGET_OPTS -O jwt-cpp-${JWT_CPP_VERSION}.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_CPP_VERSION}.tar.gz && \ - tar -xvzf jwt-cpp-${JWT_CPP_VERSION}.tar.gz && cd jwt-cpp-${JWT_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${JWT_CPP_INSTALL_DIR} && \ - rm -rf jwt-cpp-${JWT_CPP_VERSION}.tar.gz jwt-cpp-${JWT_CPP_VERSION} - -# Install ccache 4.8.1 or above -ENV CCACHE_VERSION=4.8.1 -RUN wget $WGET_OPTS https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && tar -xf ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && cp ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/local/bin/ \ - && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64 ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates ccache clang cmake git libidn11-dev libssl-dev ninja-build \ + pkg-config python3 ragel yasm \ + && rm -rf /var/lib/apt/lists/* COPY . /ydb-cpp-sdk WORKDIR /ydb-cpp-sdk -RUN rm -rf build - -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - cmake --preset ${SDK_PRESET} \ - -DYDB_SDK_INSTALL=ON \ - -DYDB_SDK_EXAMPLES=OFF \ - -DYDB_SDK_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ - -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - ccache --zero-stats >/dev/null \ - && cmake --build --preset default \ - && cmake --install build \ - && ccache --show-stats - -RUN rm -rf build -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - cmake --preset ${PRESET} \ - -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ - -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - ccache --zero-stats >/dev/null \ - && cmake --build --preset default --target slo-key-value \ - && ccache --show-stats +# CPM provides every non-platform library during this single SDK build. +RUN --mount=type=cache,target=/root/.cache/cpm,sharing=locked \ + --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake -S . -B build-sdk -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DYDB_SDK_DEPENDENCY_MODE=CPM \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + && cmake --build build-sdk --parallel \ + && cmake --install build-sdk + +RUN --mount=type=cache,target=/root/.cache/cpm,sharing=locked \ + --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_PREFIX_PATH=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ + && cmake --build build --target slo-key-value --parallel ENTRYPOINT ["./build/tests/slo_workloads/key_value/slo-key-value"] diff --git a/tests/slo_workloads/Dockerfile.userver b/tests/slo_workloads/Dockerfile.userver index 42b29517e65..ec9338e3858 100644 --- a/tests/slo_workloads/Dockerfile.userver +++ b/tests/slo_workloads/Dockerfile.userver @@ -1,234 +1,78 @@ # syntax=docker/dockerfile:1.7 +FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest -FROM ghcr.io/userver-framework/ubuntu-22.04-userver-base:latest AS base - -ENV DEBIAN_FRONTEND=noninteractive ARG PRESET=release-test-clang -ARG SDK_PRESET=release-clang ARG USERVER_VERSION=v2.12 - -# ccache settings consumed by the configure/build steps below. -ENV CCACHE_DIR=/root/.ccache -ENV CCACHE_MAXSIZE=2G -ENV CCACHE_COMPRESS=true -ENV CCACHE_COMPILERCHECK=content - -# Every RUN that hits the network retries on transient failures so one flake -# doesn't throw away 30 min of previous build work. -RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \ - echo 'Acquire::http::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries && \ - echo 'Acquire::https::Timeout "60";' >> /etc/apt/apt.conf.d/80-retries - -ENV WGET_OPTS="--tries=5 --waitretry=15 --timeout=60 --retry-connrefused --retry-on-http-error=500,502,503,504" -ENV YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk -ENV YDB_DEPS_PREFIX_PATH="${YDB_CPP_SDK_INSTALL_PREFIX};/root/ydb_deps/absl;/root/ydb_deps/protobuf;/root/ydb_deps/grpc;/root/ydb_deps/brotli" - -# Install software-properties-common and add the gcc-13 PPA with shell-level -# retry loop — Acquire::Retries doesn't cover TCP connect timeouts. -RUN for i in 1 2 3 4 5; do \ - apt-get -y update && \ - apt-get -y install software-properties-common && \ - add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ - apt-get -y update && \ - break; \ - echo "add-apt-repository attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - done && \ - apt-cache show gcc-13 > /dev/null - -# Install C++ tools and libraries -RUN for i in 1 2 3 4 5; do \ - apt-get -y install \ - git gdb wget jq ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ - rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ - libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 && \ - break; \ - echo "apt-get install attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - apt-get -y update || true; \ - done && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Install CMake -ENV CMAKE_VERSION=3.27.7 -RUN wget $WGET_OPTS https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ - -q -O cmake-install.sh \ - && chmod u+x cmake-install.sh \ - && ./cmake-install.sh --skip-license --prefix=/usr/local \ - && rm cmake-install.sh - -# Install LLVM -ENV LLVM_VERSION=17 -RUN wget $WGET_OPTS https://apt.llvm.org/llvm.sh && \ - chmod u+x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} && \ - rm llvm.sh - -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-${LLVM_VERSION} 10000 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 10000 - -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000 - -# Install abseil-cpp -ENV ABSEIL_CPP_VERSION=20230802.0 -ENV ABSEIL_CPP_INSTALL_DIR=/root/ydb_deps/absl -RUN wget $WGET_OPTS -O abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSEIL_CPP_VERSION}.tar.gz && \ - tar -xvzf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz && cd abseil-cpp-${ABSEIL_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DABSL_PROPAGATE_CXX_STD=ON .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${ABSEIL_CPP_INSTALL_DIR} && \ - rm -rf abseil-cpp-${ABSEIL_CPP_VERSION}.tar.gz abseil-cpp-${ABSEIL_CPP_VERSION} - -# Install protobuf -ENV PROTOBUF_VERSION=25.0 -ENV PROTOBUF_INSTALL_DIR=/root/ydb_deps/protobuf -RUN wget $WGET_OPTS -O protobuf-${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz && \ - tar -xvzf protobuf-${PROTOBUF_VERSION}.tar.gz && cd protobuf-${PROTOBUF_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_INSTALL=ON -Dprotobuf_ABSL_PROVIDER=package .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${PROTOBUF_INSTALL_DIR} && \ - rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} - -# Install grpc -ENV GRPC_VERSION=1.60.2 -ENV GRPC_INSTALL_DIR=/root/ydb_deps/grpc -RUN wget $WGET_OPTS -O grpc-${GRPC_VERSION}.tar.gz https://github.com/grpc/grpc/archive/refs/tags/v${GRPC_VERSION}.tar.gz && \ - tar -xvzf grpc-${GRPC_VERSION}.tar.gz && cd grpc-${GRPC_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_INSTALL_DIR};${PROTOBUF_INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ - -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${GRPC_INSTALL_DIR} && \ - rm -rf grpc-${GRPC_VERSION}.tar.gz grpc-${GRPC_VERSION} - -# Install base64 -ENV BASE64_VERSION=0.5.2 -ENV BASE64_INSTALL_DIR=/root/ydb_deps/base64 -RUN wget $WGET_OPTS -O base64-${BASE64_VERSION}.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v${BASE64_VERSION}.tar.gz && \ - tar -xvzf base64-${BASE64_VERSION}.tar.gz && cd base64-${BASE64_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${BASE64_INSTALL_DIR} && \ - rm -rf base64-${BASE64_VERSION}.tar.gz base64-${BASE64_VERSION} - -# Install brotli -ENV BROTLI_VERSION=1.1.0 -ENV BROTLI_INSTALL_DIR=/root/ydb_deps/brotli -RUN wget $WGET_OPTS -O brotli-${BROTLI_VERSION}.tar.gz https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz && \ - tar -xvzf brotli-${BROTLI_VERSION}.tar.gz && cd brotli-${BROTLI_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${BROTLI_INSTALL_DIR} .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release && \ - rm -rf brotli-${BROTLI_VERSION}.tar.gz brotli-${BROTLI_VERSION} - -# Install jwt-cpp -ENV JWT_CPP_VERSION=0.7.0 -ENV JWT_CPP_INSTALL_DIR=/root/ydb_deps/jwt-cpp -RUN wget $WGET_OPTS -O jwt-cpp-${JWT_CPP_VERSION}.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v${JWT_CPP_VERSION}.tar.gz && \ - tar -xvzf jwt-cpp-${JWT_CPP_VERSION}.tar.gz && cd jwt-cpp-${JWT_CPP_VERSION} && \ - mkdir build && cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. && \ - cmake --build . --config Release && \ - cmake --install . --config Release --prefix ${JWT_CPP_INSTALL_DIR} && \ - rm -rf jwt-cpp-${JWT_CPP_VERSION}.tar.gz jwt-cpp-${JWT_CPP_VERSION} - -# Install ccache -ENV CCACHE_VERSION=4.8.1 -RUN wget $WGET_OPTS https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && tar -xf ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz \ - && cp ccache-${CCACHE_VERSION}-linux-x86_64/ccache /usr/local/bin/ \ - && rm -rf ccache-${CCACHE_VERSION}-linux-x86_64 ccache-${CCACHE_VERSION}-linux-x86_64.tar.xz - -# Prefer the release-attached Debian package. If a matching asset is absent, -# fall back to building the package from the same pinned userver tag. +ENV DEBIAN_FRONTEND=noninteractive \ + CPM_SOURCE_CACHE=/root/.cache/cpm \ + CCACHE_DIR=/root/.ccache \ + CCACHE_MAXSIZE=2G \ + YDB_CPP_SDK_INSTALL_PREFIX=/opt/ydb-cpp-sdk + +RUN apt-get -o Dir::Etc::sourcelist=/etc/apt/sources.list \ + -o Dir::Etc::sourceparts=- update \ + && apt-get install -y --no-install-recommends \ + ca-certificates ccache cmake git jq libidn11-dev libssl-dev ninja-build pkg-config \ + python3 ragel wget yasm \ + && apt-get -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu-toolchain-r.list \ + -o Dir::Etc::sourceparts=- update \ + && apt-get install -y --no-install-recommends gcc-13 g++-13 \ + && rm -rf /var/lib/apt/lists/* + +ENV CC=gcc-13 CXX=g++-13 + +# Install the userver development package and retain its source tree only for +# chaotic code generation used by the workload. RUN set -eux; \ - release_json=/tmp/userver-release.json; \ - wget $WGET_OPTS -qO "${release_json}" \ - "https://api.github.com/repos/userver-framework/userver/releases/tags/${USERVER_VERSION}" || touch "${release_json}"; \ - USERVER_DEB_URL="$( \ - jq -r '.assets[]?.browser_download_url // empty' "${release_json}" 2>/dev/null | \ - grep -E 'libuserver-all-dev.*ubuntu.*22\.04.*amd64.*\.deb' | \ - head -n1 || true \ - )"; \ - if [ -n "${USERVER_DEB_URL}" ]; then \ - wget $WGET_OPTS -O /tmp/libuserver-all-dev.deb "${USERVER_DEB_URL}"; \ + wget -qO /tmp/release.json \ + "https://api.github.com/repos/userver-framework/userver/releases/tags/${USERVER_VERSION}" || true; \ + deb_url="$(jq -r '.assets[]?.browser_download_url // empty' /tmp/release.json | \ + grep -E 'libuserver-all-dev.*ubuntu.*22\.04.*amd64.*\.deb' | head -n1 || true)"; \ + if [ -n "$deb_url" ]; then \ + wget -qO /tmp/libuserver-all-dev.deb "$deb_url"; \ else \ - git clone --depth 1 --branch ${USERVER_VERSION} \ - https://github.com/userver-framework/userver.git /tmp/userver-build; \ - cd /tmp/userver-build; \ - BUILD_OPTIONS="-DUSERVER_FEATURE_YDB=OFF" ./scripts/build_and_install.sh; \ - cp libuserver-all-dev*.deb /tmp/libuserver-all-dev.deb; \ - rm -rf /tmp/userver-build; \ + git clone --depth 1 --branch ${USERVER_VERSION} \ + https://github.com/userver-framework/userver.git /tmp/userver-build; \ + cd /tmp/userver-build; \ + BUILD_OPTIONS="-DUSERVER_FEATURE_YDB=OFF" ./scripts/build_and_install.sh; \ + cp libuserver-all-dev*.deb /tmp/libuserver-all-dev.deb; \ fi; \ - for i in 1 2 3 4 5; do \ - apt-get -y update && \ - apt-get -y install --no-install-recommends /tmp/libuserver-all-dev.deb && \ - break; \ - echo "userver .deb install attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - done; \ - rm -rf /tmp/libuserver-all-dev.deb "${release_json}" /var/lib/apt/lists/*; \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 10000; \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 10000; \ - update-alternatives --set gcc /usr/bin/gcc-13; \ - update-alternatives --set g++ /usr/bin/g++-13 - -# Userver source checkout for chaotic codegen tools used during configure. -RUN for i in 1 2 3 4 5; do \ - git clone --depth 1 --branch ${USERVER_VERSION} \ - https://github.com/userver-framework/userver.git /userver-src && \ - break; \ - echo "userver clone attempt $i failed; sleeping $((i * 15))s"; \ - sleep $((i * 15)); \ - rm -rf /userver-src; \ - done \ - && chmod +x /userver-src/chaotic/bin-dynamic-configs/chaotic-gen-dynamic-configs \ - /userver-src/chaotic/bin/chaotic-gen 2>/dev/null || true + apt-get -o Dir::Etc::sourcelist=/etc/apt/sources.list \ + -o Dir::Etc::sourceparts=- update; \ + apt-get install -y --no-install-recommends /tmp/libuserver-all-dev.deb; \ + git clone --depth 1 --branch ${USERVER_VERSION} \ + https://github.com/userver-framework/userver.git /userver-src; \ + rm -rf /tmp/libuserver-all-dev.deb /tmp/release.json /tmp/userver-build \ + /var/lib/apt/lists/* ENV USERVER_SOURCE_DIR=/userver-src COPY . /ydb-cpp-sdk WORKDIR /ydb-cpp-sdk -RUN rm -rf build - -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - cmake --preset ${SDK_PRESET} \ - -DYDB_SDK_INSTALL=ON \ - -DYDB_SDK_EXAMPLES=OFF \ - -DYDB_SDK_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ - -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - ccache --zero-stats >/dev/null \ - && cmake --build --preset default \ - && cmake --install build \ - && ccache --show-stats - -RUN rm -rf build -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - cmake --preset ${PRESET} \ - -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ - -DCMAKE_PREFIX_PATH="${YDB_DEPS_PREFIX_PATH}" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DYDB_CPP_SDK_BUILD_SLO_USERVER=ON -RUN --mount=type=cache,target=/root/.ccache,sharing=locked \ - ccache --zero-stats >/dev/null \ - && cmake --build --preset default --target slo-userver-key-value \ - && ccache --show-stats +RUN --mount=type=cache,target=/root/.cache/cpm,sharing=locked \ + --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake -S . -B build-sdk -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DYDB_SDK_DEPENDENCY_MODE=CPM \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + && cmake --build build-sdk --parallel \ + && cmake --install build-sdk + +RUN --mount=type=cache,target=/root/.cache/cpm,sharing=locked \ + --mount=type=cache,target=/root/.ccache,sharing=locked \ + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_PREFIX_PATH=${YDB_CPP_SDK_INSTALL_PREFIX} \ + -DYDB_CPP_SDK_SLO_USE_INSTALLED_SDK=ON \ + -DYDB_CPP_SDK_BUILD_SLO_USERVER=ON \ + && cmake --build build --target slo-userver-key-value --parallel ENTRYPOINT ["./build/tests/slo_workloads/userver/key_value/slo-userver-key-value"] diff --git a/tests/slo_workloads/key_value/CMakeLists.txt b/tests/slo_workloads/key_value/CMakeLists.txt index d3c87cb9209..452730b2ec7 100644 --- a/tests/slo_workloads/key_value/CMakeLists.txt +++ b/tests/slo_workloads/key_value/CMakeLists.txt @@ -33,7 +33,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") ) elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_options(slo-key-value PRIVATE - -Wl,-platform_version,macos,11.0,11.0 -framework CoreFoundation ) diff --git a/tests/slo_workloads/utils/CMakeLists.txt b/tests/slo_workloads/utils/CMakeLists.txt index 4dd24019a27..d8a63c2dfbd 100644 --- a/tests/slo_workloads/utils/CMakeLists.txt +++ b/tests/slo_workloads/utils/CMakeLists.txt @@ -1,16 +1,3 @@ -include(FetchContent) - -FetchContent_Declare( - hdr_histogram - GIT_REPOSITORY https://github.com/HdrHistogram/HdrHistogram_c.git - GIT_TAG 0.11.8 - EXCLUDE_FROM_ALL -) -set(HDR_HISTOGRAM_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE) -set(HDR_HISTOGRAM_BUILD_SHARED OFF CACHE BOOL "" FORCE) -set(HDR_LOG_REQUIRED OFF CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(hdr_histogram) - # slo-utils-base: shared helpers, no DoMain. The userver workload links this # and supplies its own DoMain that wraps command dispatch in RunStandalone. add_library(slo-utils-base) diff --git a/third_party/FastLZ b/third_party/FastLZ deleted file mode 160000 index 344eb4025f9..00000000000 --- a/third_party/FastLZ +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 344eb4025f9ae866ebf7a2ec48850f7113a97a42 diff --git a/third_party/README.md b/third_party/README.md index 380449c2911..918eb5718be 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -1,21 +1,4 @@ -# Third-party dependencies +# Third-party sources -Vendored libraries used by the YDB C++ SDK. Initialize submodules before building: - -```bash -git submodule update --init third_party/aklomp-base64 third_party/jwt-cpp \ - third_party/opentelemetry-cpp third_party/api-common-protos third_party/FastLZ -``` - -| Dependency | Location | Version / pin | Debian package | -|------------|----------|---------------|----------------| -| aklomp-base64 | `third_party/aklomp-base64` | v0.5.2 (submodule) | `libydb-cpp-dev` | -| jwt-cpp | `third_party/jwt-cpp` | submodule tag | `libydb-cpp-dev` | -| picojson | bundled in `jwt-cpp/include/picojson` | (via jwt-cpp) | `libydb-cpp-dev` | -| opentelemetry-cpp | `third_party/opentelemetry-cpp` | v1.26.0 (submodule) | `libydb-cpp-otel-metrics-dev` | -| api-common-protos | `third_party/api-common-protos` | submodule | built into core / system package | -| FastLZ | `third_party/FastLZ` | submodule | sources only (linked into core) | -| nayuki_md5 | `third_party/nayuki_md5` | in-tree | sources only (linked into core) | - -OpenTelemetry is only required when `YDB_SDK_ENABLE_OTEL_METRICS` or `YDB_SDK_ENABLE_OTEL_TRACE` is enabled. -The tracing plugin `.deb` depends on `libydb-cpp-otel-metrics-dev` for vendored OpenTelemetry artifacts. +`nayuki_md5` remains in-tree. All fetched libraries and their pins are defined +in `cmake/dependencies.cmake` and managed by CPM. diff --git a/third_party/aklomp-base64 b/third_party/aklomp-base64 deleted file mode 160000 index 8bdda2d47ca..00000000000 --- a/third_party/aklomp-base64 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8bdda2d47caf8b066999c5bd01069e55bcd0d396 diff --git a/third_party/api-common-protos b/third_party/api-common-protos deleted file mode 160000 index 3332dec5277..00000000000 --- a/third_party/api-common-protos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3332dec527759859840a3a2ff108c67a54708130 diff --git a/third_party/jwt-cpp b/third_party/jwt-cpp deleted file mode 160000 index 4a537e96989..00000000000 --- a/third_party/jwt-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a537e969891dde542ad8b1a4a214955a83be29f diff --git a/third_party/opentelemetry-cpp b/third_party/opentelemetry-cpp deleted file mode 160000 index 2d80af1b1d2..00000000000 --- a/third_party/opentelemetry-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2d80af1b1d26e300d9c0f7f51fa360f22c773523