From 83e883fbd700907025ecb4685ba656a172ef4655 Mon Sep 17 00:00:00 2001 From: bfjelds Date: Wed, 24 Jun 2026 10:32:55 -0700 Subject: [PATCH 1/3] engineering: build azl3+azl4 RPMs in parallel at task level Replace the two sequential release.yml invocations (one per distro) with a single build-rpms-parallel.yml template plus scripts/build-rpms-parallel.sh. Shared setup runs once, then both azl3 and azl4 RPMs build concurrently as background docker builds in a single task, avoiding a second ADO job and duplicated OneBranch/setup overhead. - New scripts/build-rpms-parallel.sh (executable): run both docker builds in the background (& ... wait), capture per-distro logs and exit codes, then unpack azl3 to the base artifact dir and azl4 under azl4/. Uses set -o pipefail and anchored greps so a failed make azl-version-vars cannot be masked by the grep/cut pipeline. Per-distro dest dirs and image tags prevent clobbering. - New build-rpms-parallel.yml: run-once setup (version, deps, docker, preview container) then a single task invoking the script. Distro binaries are extracted via extract-binary.sh for azl3 and azl4. - build-source.yml: amd64 and arm64 jobs call the new template once. - Remove now-unused release.yml. Artifact layout unchanged: azl3 at base path, azl4 under azl4/. Downstream download-staged.yml and consumer distro-filters are unaffected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../trident_rpms/build-rpms-parallel.yml | 70 +++++++++++++ .../stages/trident_rpms/build-source.yml | 16 +-- .../templates/stages/trident_rpms/release.yml | 96 ------------------ scripts/build-rpms-parallel.sh | 98 +++++++++++++++++++ 4 files changed, 170 insertions(+), 110 deletions(-) create mode 100644 .pipelines/templates/stages/trident_rpms/build-rpms-parallel.yml delete mode 100644 .pipelines/templates/stages/trident_rpms/release.yml create mode 100755 scripts/build-rpms-parallel.sh diff --git a/.pipelines/templates/stages/trident_rpms/build-rpms-parallel.yml b/.pipelines/templates/stages/trident_rpms/build-rpms-parallel.yml new file mode 100644 index 000000000..1dc80eb65 --- /dev/null +++ b/.pipelines/templates/stages/trident_rpms/build-rpms-parallel.yml @@ -0,0 +1,70 @@ +parameters: + - name: targetArchitecture + type: string + default: "amd64" + values: + - arm64 + - amd64 + + - name: tridentRepoDirectory + type: string + + # Base artifact directory. azl3 RPMs/binaries land here; azl4 lands under + # the azl4/ subdirectory to preserve the single-artifact dual-distro layout. + - name: artifactDirectory + type: string + +steps: + # --- Shared setup (run once for both distros) --- + - script: | + set -eux + TRIDENT_VERSION=$(python3 ./scripts/get-version.py "$(Build.BuildNumber)" --commit) + echo "##vso[task.setvariable variable=trident_version]$TRIDENT_VERSION" + displayName: "Setting Trident version" + workingDirectory: ${{ parameters.tridentRepoDirectory }} + + - task: onebranch.pipeline.version@1 + displayName: "Set build number" + inputs: + system: "Custom" + customVersion: $(trident_version) + + - script: sudo tdnf install -y moby-buildx p7zip p7zip-plugins zstd + displayName: Install native dependencies + retryCountOnTaskFailure: 3 + + - script: | + set -eux + sudo systemctl start docker + workingDirectory: ${{ parameters.tridentRepoDirectory }} + displayName: Start Docker + + - template: ../common_tasks/preview-container.yml + parameters: + dockerfilePath: ${{ parameters.tridentRepoDirectory }}/packaging/docker/Dockerfile.full + tridentSourceDirectory: ${{ parameters.tridentRepoDirectory }} + targetArchitecture: ${{ parameters.targetArchitecture }} + + # --- Build azl3 and azl4 RPMs concurrently in a single task --- + # See scripts/build-rpms-parallel.sh: both docker builds run against the + # same BuildKit daemon as background jobs, each writing to its own dest dir + # and log file with explicit exit-code checks. This avoids a second ADO job + # (and its OneBranch/checkout/setup overhead) while overlapping the + # I/O-bound parts of the two builds. + - script: | + ./scripts/build-rpms-parallel.sh \ + "$(trident_version)" \ + "${{ parameters.tridentRepoDirectory }}/packaging/docker/Dockerfile.full" \ + "${{ parameters.artifactDirectory }}" \ + "$(Agent.TempDirectory)/_rpm_parallel" + workingDirectory: ${{ parameters.tridentRepoDirectory }} + displayName: Build azl3 + azl4 RPMs (parallel) + + # --- Extract Trident binaries for each distro --- + - script: ./scripts/extract-binary.sh ${{ parameters.artifactDirectory }} ${{ parameters.artifactDirectory }} ${{ parameters.targetArchitecture }} azl3 + workingDirectory: ${{ parameters.tridentRepoDirectory }} + displayName: Extract Trident binary (azl3) + + - script: ./scripts/extract-binary.sh ${{ parameters.artifactDirectory }}/azl4 ${{ parameters.artifactDirectory }}/azl4 ${{ parameters.targetArchitecture }} azl4 + workingDirectory: ${{ parameters.tridentRepoDirectory }} + displayName: Extract Trident binary (azl4) diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index 2b12e9756..d772096bc 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -89,17 +89,11 @@ stages: - template: ../common_tasks/cargo-auth.yml parameters: cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml - - template: release.yml + - template: build-rpms-parallel.yml parameters: targetArchitecture: ${{ parameters.targetArchitecture }} tridentRepoDirectory: $(TRIDENT_SOURCE_DIR) artifactDirectory: "$(ob_outputDirectory)" - - template: release.yml - parameters: - targetArchitecture: ${{ parameters.targetArchitecture }} - tridentRepoDirectory: $(TRIDENT_SOURCE_DIR) - artifactDirectory: "$(ob_outputDirectory)/azl4" - distro: azl4 - ${{ elseif eq( parameters.targetArchitecture, 'arm64' ) }}: - job: BuildTrident_${{ parameters.targetArchitecture }} @@ -132,14 +126,8 @@ stages: set -eux sudo systemctl start docker displayName: Start Docker - - template: release.yml + - template: build-rpms-parallel.yml parameters: targetArchitecture: ${{ parameters.targetArchitecture }} tridentRepoDirectory: $(TRIDENT_SOURCE_DIR) artifactDirectory: "$(ob_outputDirectory)" - - template: release.yml - parameters: - targetArchitecture: ${{ parameters.targetArchitecture }} - tridentRepoDirectory: $(TRIDENT_SOURCE_DIR) - artifactDirectory: "$(ob_outputDirectory)/azl4" - distro: azl4 diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml deleted file mode 100644 index 0cd2cd86a..000000000 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ /dev/null @@ -1,96 +0,0 @@ -parameters: - - name: targetArchitecture - type: string - default: "amd64" - values: - - arm64 - - amd64 - - - name: distro - type: string - default: "azl3" - values: - - azl3 - - azl4 - - - name: tridentRepoDirectory - type: string - - - name: artifactDirectory - type: string - -steps: - - script: | - set -eux - TRIDENT_VERSION=$(python3 ./scripts/get-version.py "$(Build.BuildNumber)" --commit) - echo "##vso[task.setvariable variable=trident_version]$TRIDENT_VERSION" - displayName: "Setting Trident version" - workingDirectory: ${{ parameters.tridentRepoDirectory }} - - - task: onebranch.pipeline.version@1 - displayName: "Set build number" - inputs: - system: "Custom" - customVersion: $(trident_version) - - - script: sudo tdnf install -y moby-buildx p7zip p7zip-plugins zstd - displayName: Install native dependencies - retryCountOnTaskFailure: 3 - - - script: | - set -eux - sudo systemctl start docker - workingDirectory: ${{ parameters.tridentRepoDirectory }} - displayName: Start Docker - - - template: ../common_tasks/preview-container.yml - parameters: - dockerfilePath: ${{ parameters.tridentRepoDirectory }}/packaging/docker/Dockerfile.full - tridentSourceDirectory: ${{ parameters.tridentRepoDirectory }} - targetArchitecture: ${{ parameters.targetArchitecture }} - - - script: | - set -eux - full_version=$(trident_version) - - # Separate into version and prerelease identifier - # for the RPM build. - version=$(echo $full_version | cut -d'-' -f1) - prerelease=$(echo $full_version | cut -d'-' -f2-) - - # Build RPMs and export only the artifact tarball (no image load/unpack). - # CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN is populated by the CargoAuthenticate task. - outdir="/tmp/_rpm_artifacts" - rm -rf "$outdir" - mkdir -p "$outdir" - - AZL_IMAGE="$(DISTRO=${{ parameters.distro }} make azl-version-vars | grep AZL_IMAGE | cut -d '=' -f2)" - DISTRO_PACKAGES="$(DISTRO=${{ parameters.distro }} make azl-version-vars | grep DISTRO_PACKAGES | cut -d '=' -f2)" - RPM_PACKAGES="$(DISTRO=${{ parameters.distro }} make azl-version-vars | grep RPM_PACKAGES | cut -d '=' -f2)" - RPM_DEST="$(DISTRO=${{ parameters.distro }} make azl-version-vars | grep RPM_DEST | cut -d '=' -f2)" - RUST_PACKAGE="$(DISTRO=${{ parameters.distro }} make azl-version-vars | grep RUST_PACKAGE | cut -d '=' -f2)" - - docker build -f ${{ parameters.tridentRepoDirectory }}/packaging/docker/Dockerfile.full -t trident/trident-build:latest \ - --secret id=registry_token,env=CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN \ - --build-arg TRIDENT_VERSION="$full_version" \ - --build-arg RPM_VER="$version"\ - --build-arg RPM_REL="$prerelease.${{ parameters.distro }}"\ - --build-arg AZL_IMAGE="$AZL_IMAGE"\ - --build-arg DISTRO_PACKAGES="$DISTRO_PACKAGES" \ - --build-arg RPM_PACKAGES="$RPM_PACKAGES" \ - --build-arg RUST_PACKAGE="$RUST_PACKAGE" \ - --build-arg RPM_DEST="$RPM_DEST" \ - --target artifact \ - --output type=local,dest="$outdir" \ - . - - cp -f "$outdir/trident-rpms.tar.gz" ./trident-rpms.tar.gz - - mkdir -p ${{ parameters.artifactDirectory }} - tar -xzf trident-rpms.tar.gz -C ${{ parameters.artifactDirectory }} --strip-components=3 - workingDirectory: ${{ parameters.tridentRepoDirectory }} - displayName: Build RPMs - - - script: ./scripts/extract-binary.sh ${{ parameters.artifactDirectory }} ${{ parameters.artifactDirectory }} ${{ parameters.targetArchitecture }} ${{ parameters.distro }} - workingDirectory: ${{ parameters.tridentRepoDirectory }} - displayName: Extract Trident binary diff --git a/scripts/build-rpms-parallel.sh b/scripts/build-rpms-parallel.sh new file mode 100755 index 000000000..0f393ea47 --- /dev/null +++ b/scripts/build-rpms-parallel.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Builds the azl3 and azl4 Trident RPMs concurrently against a single +# BuildKit daemon, then unpacks each into the artifact layout (azl3 at the +# base directory, azl4 under azl4/). Running both docker builds as background +# jobs in one task avoids a second ADO job and the duplicated OneBranch/setup +# overhead while overlapping the I/O-bound parts of the two builds. +# +# Usage: +# build-rpms-parallel.sh +# +# Args: +# full_version Full Trident version string (e.g. from get-version.py). +# dockerfile Path to Dockerfile.full. +# artifact_dir Base artifact directory (azl3 lands here, azl4 under azl4/). +# work_dir Scratch directory for per-distro build output and logs. +# Should be job-scoped (e.g. $(Agent.TempDirectory)/...). +# +# Expects CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN in the environment +# (populated by the CargoAuthenticate task) for the docker build secret. + +set -euxo pipefail + +full_version=$1 +dockerfile=$2 +artifact_dir=$3 +work_dir=$4 + +# Separate into version and prerelease identifier for the RPM build. +version=$(echo "$full_version" | cut -d'-' -f1) +prerelease=$(echo "$full_version" | cut -d'-' -f2-) + +build_one() { + # args: + local distro="$1" + local dest="$2" + local log="$3" + + # Resolve all distro vars from a single make invocation so a make failure + # isn't masked by the grep/cut pipeline (pipefail also guards). + local vars azl_image distro_packages rpm_packages rpm_dest rust_package + vars="$(DISTRO=$distro make azl-version-vars)" + azl_image="$(echo "$vars" | grep '^AZL_IMAGE=' | cut -d '=' -f2)" + distro_packages="$(echo "$vars" | grep '^DISTRO_PACKAGES=' | cut -d '=' -f2)" + rpm_packages="$(echo "$vars" | grep '^RPM_PACKAGES=' | cut -d '=' -f2)" + rpm_dest="$(echo "$vars" | grep '^RPM_DEST=' | cut -d '=' -f2)" + rust_package="$(echo "$vars" | grep '^RUST_PACKAGE=' | cut -d '=' -f2)" + + rm -rf "$dest" + mkdir -p "$dest" + + # Per-distro image tag avoids collisions between the two concurrent builds. + docker build -f "$dockerfile" -t "trident/trident-build:$distro" \ + --secret id=registry_token,env=CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN \ + --build-arg TRIDENT_VERSION="$full_version" \ + --build-arg RPM_VER="$version" \ + --build-arg RPM_REL="$prerelease.$distro" \ + --build-arg AZL_IMAGE="$azl_image" \ + --build-arg DISTRO_PACKAGES="$distro_packages" \ + --build-arg RPM_PACKAGES="$rpm_packages" \ + --build-arg RUST_PACKAGE="$rust_package" \ + --build-arg RPM_DEST="$rpm_dest" \ + --target artifact \ + --output type=local,dest="$dest" \ + . > "$log" 2>&1 +} + +rm -rf "$work_dir" +mkdir -p "$work_dir" + +# Launch both builds in the background. +build_one azl3 "$work_dir/azl3" "$work_dir/azl3.log" & +pid_azl3=$! +build_one azl4 "$work_dir/azl4" "$work_dir/azl4.log" & +pid_azl4=$! + +# Wait for each and capture exit codes (don't let set -e abort early). +rc_azl3=0 +rc_azl4=0 +wait "$pid_azl3" || rc_azl3=$? +wait "$pid_azl4" || rc_azl4=$? + +# Surface both logs sequentially so concurrent output is readable. +echo "===== azl3 build log =====" +cat "$work_dir/azl3.log" || true +echo "===== azl4 build log =====" +cat "$work_dir/azl4.log" || true + +if [ "$rc_azl3" -ne 0 ] || [ "$rc_azl4" -ne 0 ]; then + echo "Build failed: azl3 rc=$rc_azl3, azl4 rc=$rc_azl4" + exit 1 +fi + +# Unpack azl3 to the base artifact dir, azl4 under azl4/. +mkdir -p "$artifact_dir" +mkdir -p "$artifact_dir/azl4" +tar -xzf "$work_dir/azl3/trident-rpms.tar.gz" -C "$artifact_dir" --strip-components=3 +tar -xzf "$work_dir/azl4/trident-rpms.tar.gz" -C "$artifact_dir/azl4" --strip-components=3 From e42148b97da576240836f299b4ae1de7a734466c Mon Sep 17 00:00:00 2001 From: bfjelds Date: Wed, 24 Jun 2026 17:22:08 -0700 Subject: [PATCH 2/3] engineering: address PR review on parallel RPM build - build-source.yml: drop the redundant arm64 'Start Docker' step; the build-rpms-parallel.yml shared setup already starts Docker for both arches. - build-rpms-parallel.sh: validate arg count and refuse an empty, root, or non-absolute work_dir before the rm -rf guard. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/trident_rpms/build-source.yml | 4 ---- scripts/build-rpms-parallel.sh | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index d772096bc..78232ba13 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -122,10 +122,6 @@ stages: - template: ../common_tasks/cargo-auth.yml parameters: cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml - - script: | - set -eux - sudo systemctl start docker - displayName: Start Docker - template: build-rpms-parallel.yml parameters: targetArchitecture: ${{ parameters.targetArchitecture }} diff --git a/scripts/build-rpms-parallel.sh b/scripts/build-rpms-parallel.sh index 0f393ea47..2ee857dfa 100755 --- a/scripts/build-rpms-parallel.sh +++ b/scripts/build-rpms-parallel.sh @@ -21,11 +21,30 @@ set -euxo pipefail +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + full_version=$1 dockerfile=$2 artifact_dir=$3 work_dir=$4 +# Guardrail: work_dir is rm -rf'd below, so refuse empty, root, or any +# non-absolute path to avoid deleting an unintended directory. +case "$work_dir" in + "" | "/") + echo "Refusing to use unsafe work_dir: '$work_dir'" >&2 + exit 1 + ;; + /*) ;; + *) + echo "work_dir must be an absolute path: '$work_dir'" >&2 + exit 1 + ;; +esac + # Separate into version and prerelease identifier for the RPM build. version=$(echo "$full_version" | cut -d'-' -f1) prerelease=$(echo "$full_version" | cut -d'-' -f2-) From 6aa4e5c199ceba2691fb03ff01401a3b3f07a06a Mon Sep 17 00:00:00 2001 From: bfjelds Date: Wed, 24 Jun 2026 21:33:49 -0700 Subject: [PATCH 3/3] engineering: gate xtrace in parallel RPM build script The two builds run as concurrent background subshells; unconditional set -x interleaved trace lines from both on the task stderr, undermining the per-distro log files. Default to no xtrace and enable it only when BUILD_RPMS_DEBUG=1. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/build-rpms-parallel.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/build-rpms-parallel.sh b/scripts/build-rpms-parallel.sh index 2ee857dfa..f3e9194c6 100755 --- a/scripts/build-rpms-parallel.sh +++ b/scripts/build-rpms-parallel.sh @@ -19,7 +19,14 @@ # Expects CARGO_REGISTRIES_BMP_PUBLICPACKAGES_TOKEN in the environment # (populated by the CargoAuthenticate task) for the docker build secret. -set -euxo pipefail +set -euo pipefail + +# The two builds run as concurrent background subshells. Unconditional xtrace +# (set -x) would interleave trace lines from both on the task's stderr and +# undermine the per-distro log files, so gate it behind an explicit env var. +if [ "${BUILD_RPMS_DEBUG:-}" = "1" ]; then + set -x +fi if [ "$#" -ne 4 ]; then echo "Usage: $0 " >&2