diff --git a/.github/scripts/build-slo-image.sh b/.github/scripts/build-slo-image.sh index cd2972b0e..7aee21c0d 100755 --- a/.github/scripts/build-slo-image.sh +++ b/.github/scripts/build-slo-image.sh @@ -1,22 +1,4 @@ #!/usr/bin/env bash -# -# Builds the Docker image for the YDB Java SDK SLO workload. -# -# The script assembles a temporary build context containing two checkouts -# side by side — the SDK source tree and the ydb-java-examples checkout — -# and feeds that context to `docker build` using the Dockerfile shipped -# inside `ydb-java-examples/slo/`. -# -# The Dockerfile takes care of building the SDK from source, installing it -# into an in-image local Maven repository, and then building the workload -# against that exact SDK version. So the script does not need any Maven / -# JDK setup on the host; only `docker` and standard POSIX tools. -# -# If the initial build fails and `--fallback-image` is provided, the script -# tags the fallback image as `--tag` and exits successfully. This mirrors -# the behaviour of the equivalent script in `ydb-go-sdk` and is used by the -# baseline build, where we want to keep going even if the historical commit -# can't be built any more. set -euo pipefail @@ -26,19 +8,17 @@ Usage: build-slo-image.sh \ --sdk \ --examples \ + --workload \ --tag \ [--fallback-image ] Options: --sdk Path to the ydb-java-sdk checkout to build against. - --examples Path to the ydb-java-examples checkout that owns the - SLO workload sources (must contain slo/Dockerfile). - --tag Docker tag to assign to the built image - (e.g. ydb-app-current). + --examples Path to the ydb-java-examples checkout. + --workload SLO workload module to build. + --tag Docker tag to assign to the built image. --fallback-image If the initial Docker build fails, tag this image as - --tag and exit successfully. Useful for the baseline - build, which may be unable to compile a historical - SDK commit. + --tag and exit successfully. EOF } @@ -49,6 +29,7 @@ die() { sdk_dir="" examples_dir="" +workload="" tag="" fallback_image="" @@ -62,6 +43,10 @@ while [[ $# -gt 0 ]]; do examples_dir="${2:-}" shift 2 ;; + --workload) + workload="${2:-}" + shift 2 + ;; --tag) tag="${2:-}" shift 2 @@ -80,40 +65,61 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "$sdk_dir" || -z "$examples_dir" || -z "$tag" ]]; then +if [[ -z "$sdk_dir" || -z "$examples_dir" || -z "$workload" || -z "$tag" ]]; then usage die "Incomplete argument set" fi +workload_module="" +workload_jar="" +copy_libs="false" + +case "$workload" in + query) + workload_module="slo-workload/query" + workload_jar="ydb-slo-query-workload.jar" + copy_libs="true" + ;; + jdbc) + workload_module="slo-workload/jdbc" + workload_jar="ydb-slo-jdbc-workload.jar" + copy_libs="true" + ;; + spring-data-jdbc) + workload_module="slo-workload/spring-data-jdbc" + workload_jar="ydb-slo-spring-data-jdbc-workload.jar" + ;; + spring-data-jpa) + workload_module="slo-workload/spring-data-jpa" + workload_jar="ydb-slo-spring-data-jpa-workload.jar" + ;; + *) + die "Unsupported workload: $workload" + ;; +esac + [[ -d "$sdk_dir" ]] || die "--sdk does not exist: $sdk_dir" [[ -d "$examples_dir" ]] || die "--examples does not exist: $examples_dir" sdk_dir="$(cd "$sdk_dir" && pwd)" examples_dir="$(cd "$examples_dir" && pwd)" -dockerfile="${examples_dir}/slo/Dockerfile" +dockerfile="${examples_dir}/slo-workload/docker/Dockerfile.sdk" [[ -f "$dockerfile" ]] || die "Dockerfile not found: $dockerfile" -# Assemble a build context that contains the two checkouts side by side. -# We use hard links where possible to avoid copying large amounts of data; -# `cp -al` falls back to a regular copy when hard links aren't supported -# (e.g. across filesystems on the GitHub runner cache). context_dir="$(mktemp -d)" trap 'rm -rf "$context_dir"' EXIT echo "Assembling build context in $context_dir" echo " ydb-java-sdk: $sdk_dir" echo " ydb-java-examples: $examples_dir" +echo " workload: $workload" echo " tag: $tag" copy_tree() { local src="$1" local dst="$2" mkdir -p "$dst" - # The "/." suffix on src and "/" on dst asks cp to copy CONTENTS of src - # into dst, regardless of whether dst pre-existed. Without this, partial - # hardlink failures can leave dst partially populated and the fallback - # then nests src inside dst (dst/src/...) instead of overwriting. if cp -al "$src"/. "$dst"/ 2>/dev/null; then return 0 fi @@ -123,26 +129,21 @@ copy_tree() { copy_tree "$sdk_dir" "$context_dir/ydb-java-sdk" copy_tree "$examples_dir" "$context_dir/ydb-java-examples" -# Drop .git from each copied tree so it doesn't ship into image layers or -# confuse Maven plugins that probe for git metadata. rm -rf "$context_dir/ydb-java-sdk/.git" "$context_dir/ydb-java-examples/.git" -# Fail fast with a clear message if the layout is wrong (e.g. because of a -# silent copy failure on the runner). for required in \ "$context_dir/ydb-java-sdk/pom.xml" \ - "$context_dir/ydb-java-examples/slo/Dockerfile" + "$context_dir/ydb-java-examples/slo-workload/docker/Dockerfile.sdk" do [[ -f "$required" ]] || die "Build context missing required file: $required" done -echo "Build context layout:" -ls -la "$context_dir" -echo " ydb-java-sdk: $(ls -1 "$context_dir/ydb-java-sdk" | wc -l) entries" - set +e docker build \ --platform linux/amd64 \ + --build-arg "WORKLOAD_MODULE=${workload_module}" \ + --build-arg "WORKLOAD_JAR=${workload_jar}" \ + --build-arg "COPY_LIBS=${copy_libs}" \ -t "$tag" \ -f "$dockerfile" \ "$context_dir" diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index e5bdf1eae..cacc939fb 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -17,12 +17,18 @@ jobs: strategy: fail-fast: false matrix: - sdk: + workload: - name: java-query-kv - command: "" + module: query + - name: java-jdbc-kv + module: jdbc + - name: java-spring-data-jdbc-kv + module: spring-data-jdbc + - name: java-spring-data-jpa-kv + module: spring-data-jpa concurrency: - group: slo-${{ github.ref }}-${{ matrix.sdk.name }} + group: slo-${{ github.ref }}-${{ matrix.workload.name }} cancel-in-progress: true steps: @@ -98,20 +104,20 @@ jobs: set -euxo pipefail chmod +x sdk-current/.github/scripts/build-slo-image.sh sdk-current/.github/scripts/build-slo-image.sh \ - --sdk "${GITHUB_WORKSPACE}/sdk-current" \ - --examples "${GITHUB_WORKSPACE}/examples" \ - --tag ydb-app-current + --sdk sdk-current \ + --examples examples \ + --workload ${{ matrix.workload.module }} \ + --tag "ydb-app-current-${{ matrix.workload.module }}" - name: Build baseline workload image run: | set -euxo pipefail - # Reuse the build script from the current checkout — it doesn't - # depend on SDK contents, only on the layout it produces. sdk-current/.github/scripts/build-slo-image.sh \ - --sdk "${GITHUB_WORKSPACE}/sdk-baseline" \ - --examples "${GITHUB_WORKSPACE}/examples" \ - --tag ydb-app-baseline \ - --fallback-image ydb-app-current + --sdk sdk-baseline \ + --examples examples \ + --workload ${{ matrix.workload.module }} \ + --tag "ydb-app-baseline-${{ matrix.workload.module }}" \ + --fallback-image "ydb-app-current-${{ matrix.workload.module }}" - name: Run SLO Tests uses: ydb-platform/ydb-slo-action/init@v2 @@ -119,41 +125,41 @@ jobs: with: github_issue: ${{ github.event.pull_request.number }} github_token: ${{ secrets.GITHUB_TOKEN }} - workload_name: ${{ matrix.sdk.name }} + workload_name: ${{ matrix.workload.name }} workload_duration: "600" workload_current_ref: ${{ github.head_ref || github.ref_name }} - workload_current_image: ydb-app-current - workload_current_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100 + workload_current_image: ydb-app-current-${{ matrix.workload.module }} + workload_current_command: --read-rps 1000 --write-rps 100 workload_baseline_ref: ${{ steps.baseline.outputs.ref }} - workload_baseline_image: ydb-app-baseline - workload_baseline_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100 - -# publish-slo-report: -# needs: ydb-slo-action -# runs-on: ubuntu-latest -# name: Publish YDB SLO Report -# permissions: -# issues: write -# checks: write -# contents: read -# pull-requests: write -# steps: -# - name: Checkout current SDK version -# uses: actions/checkout@v5 -# -# - name: Publish YDB SLO Report -# uses: ydb-platform/ydb-slo-action/report@v2 -# with: -# github_token: ${{ secrets.GITHUB_TOKEN }} -# github_run_id: ${{ github.event.workflow_run.id }} -# -# - name: Remove SLO label from PR -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# PRS: ${{ toJSON(github.event.workflow_run.pull_requests) }} -# REPO: ${{ github.event.workflow_run.repository.full_name }} -# run: | -# set -euo pipefail -# PR=$(jq -r '.[0].number' <<<"$PRS") -# gh pr edit "$PR" --repo "$REPO" --remove-label SLO -# + workload_baseline_image: ydb-app-baseline-${{ matrix.workload.module }} + workload_baseline_command: --read-rps 1000 --write-rps 100 + + publish-slo-report: + needs: ydb-slo-action + if: ${{ always() && !cancelled() }} + + name: Publish YDB SLO Report + runs-on: ubuntu-latest + + permissions: + contents: read + pull-requests: write + checks: write + + steps: + - name: Publish YDB SLO Report + uses: ydb-platform/ydb-slo-action/report@v2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + github_run_id: ${{ github.run_id }} + github_issue: ${{ github.event.pull_request.number }} + + - name: Remove SLO label from PR + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh pr edit "${{ github.event.pull_request.number }}" \ + --repo "${{ github.repository }}" \ + --remove-label SLO