Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 45 additions & 44 deletions .github/scripts/build-slo-image.sh
Original file line number Diff line number Diff line change
@@ -1,22 +1,4 @@
#!/usr/bin/env bash
#
# Builds the Docker image for the YDB Java SDK SLO workload.
#
# The script assembles a temporary build context containing two checkouts
# side by side — the SDK source tree and the ydb-java-examples checkout —
# and feeds that context to `docker build` using the Dockerfile shipped
# inside `ydb-java-examples/slo/`.
#
# The Dockerfile takes care of building the SDK from source, installing it
# into an in-image local Maven repository, and then building the workload
# against that exact SDK version. So the script does not need any Maven /
# JDK setup on the host; only `docker` and standard POSIX tools.
#
# If the initial build fails and `--fallback-image` is provided, the script
# tags the fallback image as `--tag` and exits successfully. This mirrors
# the behaviour of the equivalent script in `ydb-go-sdk` and is used by the
# baseline build, where we want to keep going even if the historical commit
# can't be built any more.

set -euo pipefail

Expand All @@ -26,19 +8,17 @@ Usage:
build-slo-image.sh \
--sdk <path> \
--examples <path> \
--workload <query|jdbc|spring-data-jdbc|spring-data-jpa> \
--tag <docker-tag> \
[--fallback-image <docker-tag>]

Options:
--sdk Path to the ydb-java-sdk checkout to build against.
--examples Path to the ydb-java-examples checkout that owns the
SLO workload sources (must contain slo/Dockerfile).
--tag Docker tag to assign to the built image
(e.g. ydb-app-current).
--examples Path to the ydb-java-examples checkout.
--workload SLO workload module to build.
--tag Docker tag to assign to the built image.
--fallback-image If the initial Docker build fails, tag this image as
--tag and exit successfully. Useful for the baseline
build, which may be unable to compile a historical
SDK commit.
--tag and exit successfully.
EOF
}

Expand All @@ -49,6 +29,7 @@ die() {

sdk_dir=""
examples_dir=""
workload=""
tag=""
fallback_image=""

Expand All @@ -62,6 +43,10 @@ while [[ $# -gt 0 ]]; do
examples_dir="${2:-}"
shift 2
;;
--workload)
workload="${2:-}"
shift 2
;;
--tag)
tag="${2:-}"
shift 2
Expand All @@ -80,40 +65,61 @@ while [[ $# -gt 0 ]]; do
esac
done

if [[ -z "$sdk_dir" || -z "$examples_dir" || -z "$tag" ]]; then
if [[ -z "$sdk_dir" || -z "$examples_dir" || -z "$workload" || -z "$tag" ]]; then
usage
die "Incomplete argument set"
fi

workload_module=""
workload_jar=""
copy_libs="false"

case "$workload" in
query)
workload_module="slo-workload/query"
workload_jar="ydb-slo-query-workload.jar"
copy_libs="true"
;;
jdbc)
workload_module="slo-workload/jdbc"
workload_jar="ydb-slo-jdbc-workload.jar"
copy_libs="true"
;;
spring-data-jdbc)
workload_module="slo-workload/spring-data-jdbc"
workload_jar="ydb-slo-spring-data-jdbc-workload.jar"
;;
spring-data-jpa)
workload_module="slo-workload/spring-data-jpa"
workload_jar="ydb-slo-spring-data-jpa-workload.jar"
;;
*)
die "Unsupported workload: $workload"
;;
esac

[[ -d "$sdk_dir" ]] || die "--sdk does not exist: $sdk_dir"
[[ -d "$examples_dir" ]] || die "--examples does not exist: $examples_dir"

sdk_dir="$(cd "$sdk_dir" && pwd)"
examples_dir="$(cd "$examples_dir" && pwd)"

dockerfile="${examples_dir}/slo/Dockerfile"
dockerfile="${examples_dir}/slo-workload/docker/Dockerfile.sdk"
[[ -f "$dockerfile" ]] || die "Dockerfile not found: $dockerfile"

# Assemble a build context that contains the two checkouts side by side.
# We use hard links where possible to avoid copying large amounts of data;
# `cp -al` falls back to a regular copy when hard links aren't supported
# (e.g. across filesystems on the GitHub runner cache).
context_dir="$(mktemp -d)"
trap 'rm -rf "$context_dir"' EXIT

echo "Assembling build context in $context_dir"
echo " ydb-java-sdk: $sdk_dir"
echo " ydb-java-examples: $examples_dir"
echo " workload: $workload"
echo " tag: $tag"

copy_tree() {
local src="$1"
local dst="$2"
mkdir -p "$dst"
# The "/." suffix on src and "/" on dst asks cp to copy CONTENTS of src
# into dst, regardless of whether dst pre-existed. Without this, partial
# hardlink failures can leave dst partially populated and the fallback
# then nests src inside dst (dst/src/...) instead of overwriting.
if cp -al "$src"/. "$dst"/ 2>/dev/null; then
return 0
fi
Expand All @@ -123,26 +129,21 @@ copy_tree() {
copy_tree "$sdk_dir" "$context_dir/ydb-java-sdk"
copy_tree "$examples_dir" "$context_dir/ydb-java-examples"

# Drop .git from each copied tree so it doesn't ship into image layers or
# confuse Maven plugins that probe for git metadata.
rm -rf "$context_dir/ydb-java-sdk/.git" "$context_dir/ydb-java-examples/.git"

# Fail fast with a clear message if the layout is wrong (e.g. because of a
# silent copy failure on the runner).
for required in \
"$context_dir/ydb-java-sdk/pom.xml" \
"$context_dir/ydb-java-examples/slo/Dockerfile"
"$context_dir/ydb-java-examples/slo-workload/docker/Dockerfile.sdk"
do
[[ -f "$required" ]] || die "Build context missing required file: $required"
done

echo "Build context layout:"
ls -la "$context_dir"
echo " ydb-java-sdk: $(ls -1 "$context_dir/ydb-java-sdk" | wc -l) entries"

set +e
docker build \
--platform linux/amd64 \
--build-arg "WORKLOAD_MODULE=${workload_module}" \
--build-arg "WORKLOAD_JAR=${workload_jar}" \
--build-arg "COPY_LIBS=${copy_libs}" \
-t "$tag" \
-f "$dockerfile" \
"$context_dir"
Expand Down
100 changes: 53 additions & 47 deletions .github/workflows/slo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,18 @@ jobs:
strategy:
fail-fast: false
matrix:
sdk:
workload:
- name: java-query-kv
command: ""
module: query
- name: java-jdbc-kv
module: jdbc
- name: java-spring-data-jdbc-kv
module: spring-data-jdbc
- name: java-spring-data-jpa-kv
module: spring-data-jpa

concurrency:
group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
group: slo-${{ github.ref }}-${{ matrix.workload.name }}
cancel-in-progress: true

steps:
Expand Down Expand Up @@ -98,62 +104,62 @@ jobs:
set -euxo pipefail
chmod +x sdk-current/.github/scripts/build-slo-image.sh
sdk-current/.github/scripts/build-slo-image.sh \
--sdk "${GITHUB_WORKSPACE}/sdk-current" \
--examples "${GITHUB_WORKSPACE}/examples" \
--tag ydb-app-current
--sdk sdk-current \
--examples examples \
--workload ${{ matrix.workload.module }} \
--tag "ydb-app-current-${{ matrix.workload.module }}"

- name: Build baseline workload image
run: |
set -euxo pipefail
# Reuse the build script from the current checkout — it doesn't
# depend on SDK contents, only on the layout it produces.
sdk-current/.github/scripts/build-slo-image.sh \
--sdk "${GITHUB_WORKSPACE}/sdk-baseline" \
--examples "${GITHUB_WORKSPACE}/examples" \
--tag ydb-app-baseline \
--fallback-image ydb-app-current
--sdk sdk-baseline \
--examples examples \
--workload ${{ matrix.workload.module }} \
--tag "ydb-app-baseline-${{ matrix.workload.module }}" \
--fallback-image "ydb-app-current-${{ matrix.workload.module }}"

- name: Run SLO Tests
uses: ydb-platform/ydb-slo-action/init@v2
timeout-minutes: 30
with:
github_issue: ${{ github.event.pull_request.number }}
github_token: ${{ secrets.GITHUB_TOKEN }}
workload_name: ${{ matrix.sdk.name }}
workload_name: ${{ matrix.workload.name }}
workload_duration: "600"
workload_current_ref: ${{ github.head_ref || github.ref_name }}
workload_current_image: ydb-app-current
workload_current_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100
workload_current_image: ydb-app-current-${{ matrix.workload.module }}
workload_current_command: --read-rps 1000 --write-rps 100
workload_baseline_ref: ${{ steps.baseline.outputs.ref }}
workload_baseline_image: ydb-app-baseline
workload_baseline_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100

# publish-slo-report:
# needs: ydb-slo-action
# runs-on: ubuntu-latest
# name: Publish YDB SLO Report
# permissions:
# issues: write
# checks: write
# contents: read
# pull-requests: write
# steps:
# - name: Checkout current SDK version
# uses: actions/checkout@v5
#
# - name: Publish YDB SLO Report
# uses: ydb-platform/ydb-slo-action/report@v2
# with:
# github_token: ${{ secrets.GITHUB_TOKEN }}
# github_run_id: ${{ github.event.workflow_run.id }}
#
# - name: Remove SLO label from PR
# env:
# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# PRS: ${{ toJSON(github.event.workflow_run.pull_requests) }}
# REPO: ${{ github.event.workflow_run.repository.full_name }}
# run: |
# set -euo pipefail
# PR=$(jq -r '.[0].number' <<<"$PRS")
# gh pr edit "$PR" --repo "$REPO" --remove-label SLO
#
workload_baseline_image: ydb-app-baseline-${{ matrix.workload.module }}
workload_baseline_command: --read-rps 1000 --write-rps 100

publish-slo-report:
needs: ydb-slo-action
if: ${{ always() && !cancelled() }}

name: Publish YDB SLO Report
runs-on: ubuntu-latest

permissions:
contents: read
pull-requests: write
checks: write

steps:
- name: Publish YDB SLO Report
uses: ydb-platform/ydb-slo-action/report@v2
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
github_run_id: ${{ github.run_id }}
github_issue: ${{ github.event.pull_request.number }}

- name: Remove SLO label from PR
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
gh pr edit "${{ github.event.pull_request.number }}" \
--repo "${{ github.repository }}" \
--remove-label SLO