From ad73ad0a14889b2c053410bb9e588b4f8ed8e606 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 24 Jun 2026 23:46:22 +0000 Subject: [PATCH 1/7] azl4: rollback and servicing pipeline support Wire the rollback and servicing test pipelines to build and exercise Azure Linux 4 grub test images: shared common_tasks templates to download the azl4 base VHDX and prepare test-image requirements (including SSH key staging, which replaces the simple inline staging step from the previous change), refactored build-image and trident-testimg templates, and the rollback/servicing testing templates (skip netplan runtime testing on grubazl4; document the secure-boot gap on the azl4 base image). Stacked on the azl4 test-image PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/templates/e2e-template.yml | 2 + .../build_image/build-image-template.yml | 74 +++++---------- .../stages/build_image/build-image.yml | 12 ++- .../common_tasks/download-azl4-base-vhdx.yml | 53 +++++++++++ .../prepare-testimage-requirements.yml | 71 ++++++++++++++ .../testing_rollback/testing-template.yml | 13 ++- .../stages/testing_rollback/vm-testing.yml | 21 ++++- .../testing_servicing/testing-template.yml | 6 +- .../stages/testing_servicing/vm-testing.yml | 61 ++++++++++++ .../trident-testimg-template.yml | 92 +++++-------------- 10 files changed, 282 insertions(+), 123 deletions(-) create mode 100644 .pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml create mode 100644 .pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index a0654303a4..bf5e827c77 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -245,6 +245,7 @@ stages: # Test Runtime Updates and Rollback on VMs - template: stages/testing_rollback/vm-testing.yml parameters: + includeAzl4: true includeQemu: true includeUKI: true micBuildType: ${{ parameters.micBuildType }} @@ -260,6 +261,7 @@ stages: # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml parameters: + includeAzl4: true includeQemu: true includeUKI: true includeAzure: ${{ parameters.includeAzure }} diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 75b3521d6b..8a250bf355 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -19,6 +19,7 @@ parameters: values: - "2.0" - "3.0" + - "4.0-preview" default: "3.0" - name: micBuildType @@ -101,57 +102,32 @@ steps: # Note: DHCP packages are not being installed in this image template. # If needed use template download-dhcp.yml on test-images. - - template: common/base-images-download-template.yaml@platform-pipelines + - ${{ if ne(parameters.azureLinuxVersion, '4.0-preview') }}: + - template: common/base-images-download-template.yaml@platform-pipelines + parameters: + buildType: ${{ parameters.baseimgBuildType }} + baseImageType: $(baseImageType) + imageVersion: ${{ parameters.baseimgVersion }} + azureLinuxVersion: ${{ parameters.azureLinuxVersion }} + + # For dev builds, the RPMs are not necessarily published, so download them here. The + # parameter is set based on a runtime variable, so we cannot use ${{ if }}, so the + # template is always run and contains logic to skip if not needed. + - template: common/rpms-download-template.yaml@platform-pipelines + parameters: + rpmsVersion: ${{ parameters.rpmsVersion }} + azureLinuxVersion: ${{ parameters.azureLinuxVersion }} + runtimeBuildType: ${{ parameters.baseimgBuildType }} + - ${{ else }}: + - template: ../common_tasks/download-azl4-base-vhdx.yml + parameters: + tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} + + - template: ../common_tasks/prepare-testimage-requirements.yml parameters: - buildType: ${{ parameters.baseimgBuildType }} - baseImageType: $(baseImageType) - imageVersion: ${{ parameters.baseimgVersion }} + tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} azureLinuxVersion: ${{ parameters.azureLinuxVersion }} - - # For dev builds, the RPMs are not necessarily published, so download them here. The - # parameter is set based on a runtime variable, so we cannot use ${{ if }}, so the - # template is always run and contains logic to skip if not needed. - - template: common/rpms-download-template.yaml@platform-pipelines - parameters: - rpmsVersion: ${{ parameters.rpmsVersion }} - azureLinuxVersion: ${{ parameters.azureLinuxVersion }} - runtimeBuildType: ${{ parameters.baseimgBuildType }} - - - bash: | - set -ex - - # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx) - mkdir -p artifacts - if ls "$(Build.ArtifactStagingDirectory)/images" | grep -q ".*\.vhdx$"; then - mv $(Build.ArtifactStagingDirectory)/images/*.vhdx artifacts/ - rm -rf $(Build.ArtifactStagingDirectory)/images - else - echo "No base image found" - exit 1 - fi - - # Move dev RPM overrides to artifacts/rpm-overrides/ - if ls "$(Build.ArtifactStagingDirectory)/rpms/" 2>/dev/null | grep -q "rpms.tar.gz"; then - mkdir -p artifacts/rpm-overrides - tar -xvf $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz \ - --strip-components=2 \ - -C artifacts/rpm-overrides - rm $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz - fi - - DISTRO=azl3 - if [[ "${{ parameters.azureLinuxVersion }}" == "4.0-preview" ]]; then - DISTRO=azl4 - fi - - # Move Trident RPMs to bin/RPMS/ (builder expects bin/RPMS/*.rpm) - if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then - mkdir -p bin/RPMS - find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} bin/RPMS/ \; - rm -rf "$(Build.ArtifactStagingDirectory)/trident" - fi - displayName: "Prepare and move requirements" - workingDirectory: ${{ parameters.tridentSourceDirectory }} + stageSshKeys: ${{ eq(parameters.azureLinuxVersion, '4.0-preview') }} - bash: | set -ex diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index 873a74d034..2250cc85a8 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -24,6 +24,15 @@ parameters: type: string default: "*.*.*" + - name: azureLinuxVersion + displayName: Version of AzureLinux for the Base Image + type: string + values: + - "2.0" + - "3.0" + - "4.0-preview" + default: "3.0" + - name: clones displayName: "Number of clones to generate" type: number @@ -59,7 +68,6 @@ stages: variables: ob_outputDirectory: /tmp/output ob_artifactBaseName: ${{ parameters.imageName }} - BASEIMG_AZURE_LINUX_VERSION: "3.0" steps: - template: ../common_tasks/checkout_trident.yml @@ -84,7 +92,7 @@ stages: baseimgBuildType: $(BASEIMG_BUILD_TYPE) baseimgVersion: $(BASEIMG_VERSION) rpmsVersion: $(RPMS_VERSION) - azureLinuxVersion: ${{ variables.BASEIMG_AZURE_LINUX_VERSION }} + azureLinuxVersion: ${{ parameters.azureLinuxVersion }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: ${{ parameters.clones }} diff --git a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml new file mode 100644 index 0000000000..430cee0c30 --- /dev/null +++ b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml @@ -0,0 +1,53 @@ +parameters: + - name: tridentSourceDirectory + type: string + + # The AZL4 base VHDX is sourced from the Azure Linux preview gallery + # backing storage account. The pipeline service connection must have + # `Storage Blob Data Reader` on this account. + # See tests/images/SERVICE-CONNECTION-RUNBOOK.md. + - name: blobStorageAccount + type: string + default: "azlpubdev2mruiyvi" + + - name: blobContainer + type: string + default: "images-dev" + + - name: blobSubscription + type: string + # Subscription where the storage account lives. The service connection + # default subscription may differ, so set context before download. + default: "e4ab81f8-030f-4593-a8f2-3ea2c7630a19" + + - name: blobServiceConnection + type: string + # NB: this must be a service connection that exists in the ADO project. + # Created manually by trident infra team. + default: "trident-azl4-blob-reader" + +steps: + # Download the AZL4 base VHDX from the preview gallery backing storage. + # Authenticates via the federated identity attached to the service + # connection; no storage keys handled here. + # + # The service connection default subscription (Polar_ImageTools_Staging) + # differs from the storage account subscription (ControlTower_Test), so + # switch context so blob resolution targets the right account. + - task: AzureCLI@2 + displayName: "Download AZL4 base VHDX from blob" + inputs: + azureSubscription: ${{ parameters.blobServiceConnection }} + scriptType: bash + scriptLocation: inlineScript + workingDirectory: ${{ parameters.tridentSourceDirectory }} + inlineScript: | + set -euxo pipefail + az account set --subscription "${{ parameters.blobSubscription }}" + python3 ./tests/images/testimages.py download-image azl4_qemu_guest \ + --blob-storage-account "${{ parameters.blobStorageAccount }}" \ + --blob-container "${{ parameters.blobContainer }}" + ls -la artifacts/azl4_qemu_guest.vhdx + mkdir -p $(Build.ArtifactStagingDirectory)/images + mv artifacts/azl4_qemu_guest.vhdx $(Build.ArtifactStagingDirectory)/images/ + ls -la $(Build.ArtifactStagingDirectory)/images/azl4_qemu_guest.vhdx diff --git a/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml b/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml new file mode 100644 index 0000000000..72b839131e --- /dev/null +++ b/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml @@ -0,0 +1,71 @@ +parameters: + - name: tridentSourceDirectory + type: string + + - name: azureLinuxVersion + displayName: Version of AzureLinux for the Base Image + type: string + default: "3.0" + + - name: stageSshKeys + type: boolean + default: false + +steps: + - ${{ if eq(parameters.stageSshKeys, true) }}: + # Stage SSH keys into the testimage tree so they get baked into the image + # and are available for tests to use. + - bash: | + set -euxo pipefail + SSH_DEST="${{ parameters.tridentSourceDirectory }}/tests/images/trident-vm-testimage/base/files" + SSH_SRC="${{ parameters.tridentSourceDirectory }}/artifacts" + + if [ ! -f "$SSH_SRC/id_rsa.pub" ]; then + echo "ssh keys not found at $SSH_SRC" + echo "Available files:" + find "$SSH_SRC" -type f 2>/dev/null | head -20 || true + exit 1 + fi + + mkdir -p "$SSH_DEST" + cp "$SSH_SRC/id_rsa"* "$SSH_DEST/" + chmod 600 "$SSH_DEST/id_rsa" + chmod 644 "$SSH_DEST/id_rsa.pub" + displayName: "Stage SSH keys into testimage tree" + workingDirectory: ${{ parameters.tridentSourceDirectory }} + + - bash: | + set -ex + + # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx) + mkdir -p artifacts + if ls "$(Build.ArtifactStagingDirectory)/images" | grep -q ".*\.vhdx$"; then + mv $(Build.ArtifactStagingDirectory)/images/*.vhdx artifacts/ + rm -rf $(Build.ArtifactStagingDirectory)/images + else + echo "No base image found" + exit 1 + fi + + # Move dev RPM overrides to artifacts/rpm-overrides/ + if ls "$(Build.ArtifactStagingDirectory)/rpms/" 2>/dev/null | grep -q "rpms.tar.gz"; then + mkdir -p artifacts/rpm-overrides + tar -xvf $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz \ + --strip-components=2 \ + -C artifacts/rpm-overrides + rm $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz + fi + + DISTRO=azl3 + if [[ "${{ parameters.azureLinuxVersion }}" == "4.0-preview" ]]; then + DISTRO=azl4 + fi + + # Move Trident RPMs to bin/RPMS/ (builder expects bin/RPMS/*.rpm) + if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then + mkdir -p bin/RPMS + find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} bin/RPMS/ \; + rm -rf "$(Build.ArtifactStagingDirectory)/trident" + fi + displayName: "Prepare and move requirements" + workingDirectory: ${{ parameters.tridentSourceDirectory }} diff --git a/.pipelines/templates/stages/testing_rollback/testing-template.yml b/.pipelines/templates/stages/testing_rollback/testing-template.yml index 0077fa49e9..0083d9b8bd 100644 --- a/.pipelines/templates/stages/testing_rollback/testing-template.yml +++ b/.pipelines/templates/stages/testing_rollback/testing-template.yml @@ -25,6 +25,7 @@ parameters: type: string values: - qemu-grub + - grubazl4 - qemu - uki @@ -160,7 +161,10 @@ jobs: fi if [ "${{ parameters.flavor }}" != "uki" ]; then if [[ "${{ parameters.testSecureBoot }}" == 'True' ]]; then - STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --secure-boot" + # grubazl4 is skipped: its base image does not support secure boot yet. + if [[ "${{ parameters.flavor }}" != 'grubazl4' ]]; then + STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --secure-boot" + fi fi fi if [ "${{ parameters.skipManualRollbackTesting }}" == "true" ]; then @@ -175,7 +179,12 @@ jobs: # Allow testing for non-extension scenarios STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --skip-extension-testing" fi - if [ "${{ parameters.skipNetplanRuntimeTesting }}" == "true" ]; then + # Netplan runtime testing is skipped on grubazl4 (Azure Linux 4) to + # avoid an early-boot systemd generator deadlock during rollback. + # Fixed upstream by azurelinux PR #17791 + # (https://github.com/microsoft/azurelinux/pull/17791/). + # Force it on for grubazl4 regardless of the pipeline parameter. + if [ "${{ parameters.skipNetplanRuntimeTesting }}" == "true" ] || [ "${{ parameters.flavor }}" == "grubazl4" ]; then # skip netplan runtime testing STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --skip-netplan-runtime-testing" fi diff --git a/.pipelines/templates/stages/testing_rollback/vm-testing.yml b/.pipelines/templates/stages/testing_rollback/vm-testing.yml index cbb8482732..6aa53e7850 100644 --- a/.pipelines/templates/stages/testing_rollback/vm-testing.yml +++ b/.pipelines/templates/stages/testing_rollback/vm-testing.yml @@ -32,6 +32,11 @@ parameters: type: boolean default: false + - name: includeAzl4 + displayName: "Include qemu azl4 testing" + type: boolean + default: false + - name: includeUKI displayName: "Include UKI testing" type: boolean @@ -74,7 +79,7 @@ stages: - ${{ parameters.dependsOnStage }} jobs: - - template: ../testing_servicing/build-image.yml + - template: ../trident_images/build-image.yml parameters: label: "qemu-grub-base" makeTarget: "artifacts/trident-vm-grub-testimage.qcow2" @@ -84,7 +89,7 @@ stages: micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true - - template: ../testing_servicing/build-image.yml + - template: ../trident_images/build-image.yml parameters: label: "qemu-grub-update-a" makeTarget: "artifacts/trident-vm-grub-testimage.cosi" @@ -103,6 +108,8 @@ stages: - BuildingTools - ${{ if eq(parameters.includeQemuGrub, true) }}: - BuildImagesQemuGrub + - ${{ if eq(parameters.includeAzl4, true) }}: + - BuildImagesQemuGrubAzl4 - ${{ if eq(parameters.includeQemu, true) }}: - BuildImagesQemu - ${{ if eq(parameters.includeUKI, true) }}: @@ -128,6 +135,16 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} testSecureBoot: ${{ parameters.testSecureBoot }} + - ${{ if eq(parameters.includeAzl4, true) }}: + - template: testing-template.yml + parameters: + updateCheckTimeoutInMinutes: ${{ parameters.updateCheckTimeoutInMinutes }} + verboseLogging: ${{ parameters.verboseLogging }} + platform: qemu + flavor: grubazl4 + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + testSecureBoot: ${{ parameters.testSecureBoot }} - ${{ if eq(parameters.includeQemu, true) }}: - template: testing-template.yml parameters: diff --git a/.pipelines/templates/stages/testing_servicing/testing-template.yml b/.pipelines/templates/stages/testing_servicing/testing-template.yml index 6e222f8989..6f5f4f587b 100644 --- a/.pipelines/templates/stages/testing_servicing/testing-template.yml +++ b/.pipelines/templates/stages/testing_servicing/testing-template.yml @@ -40,6 +40,7 @@ parameters: displayName: Image flavor type: string values: + - grubazl4 - qemu - azure - uki @@ -167,7 +168,10 @@ jobs: fi if [ "${{ parameters.flavor }}" != "uki" ]; then if [[ "${{ parameters.testSecureBoot }}" == 'True' ]]; then - FLAGS="$FLAGS --secure-boot" + # grubazl4 is skipped: its base image does not support secure boot yet. + if [[ "${{ parameters.flavor }}" != 'grubazl4' ]]; then + FLAGS="$FLAGS --secure-boot" + fi fi fi if [ "${{ parameters.rollbackTesting }}" == "True" ]; then diff --git a/.pipelines/templates/stages/testing_servicing/vm-testing.yml b/.pipelines/templates/stages/testing_servicing/vm-testing.yml index 117e0bf040..6fca4f16bc 100644 --- a/.pipelines/templates/stages/testing_servicing/vm-testing.yml +++ b/.pipelines/templates/stages/testing_servicing/vm-testing.yml @@ -51,6 +51,11 @@ parameters: type: boolean default: false + - name: includeAzl4 + displayName: "Include qemu azl4 testing" + type: boolean + default: false + - name: includeAzure displayName: "Include Azure testing" type: boolean @@ -93,6 +98,46 @@ stages: jobs: - template: generate-ssh-keys.yml + - ${{ if eq(parameters.includeAzl4, true) }}: + - stage: BuildImagesQemuGrubAzl4 + displayName: Build Base and Update Images for AZL4 + dependsOn: + - PrepareSSHKeys + - GetTridentBinaries_rpms_amd64 + - ${{ if ne(parameters.dependsOnStage, '') }}: + - ${{ parameters.dependsOnStage }} + + jobs: + - template: ../trident_images/build-image.yml + parameters: + label: "grubazl4-base" + makeTarget: "artifacts/trident-vm-grub-azl4-testimage.qcow2" + baseimgAzlVersion: "4.0-preview" + baseimgType: qemu_guest + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + useStagedSshKeys: true + + - template: ../trident_images/build-image.yml + parameters: + label: "grubazl4-update-a" + makeTarget: "artifacts/trident-vm-grub-azl4-testimage.cosi" + baseimgAzlVersion: "4.0-preview" + baseimgType: qemu_guest + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + useStagedSshKeys: true + + - template: ../trident_images/build-image.yml + parameters: + label: "grubazl4-update-b" + makeTarget: "artifacts/trident-vm-grub-azl4-testimage.cosi" + baseimgAzlVersion: "4.0-preview" + baseimgType: qemu_guest + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + useStagedSshKeys: true + - ${{ if eq(parameters.includeQemu, true) }}: - stage: BuildImagesQemu displayName: Build Base and Update Images for QEMU @@ -223,6 +268,8 @@ stages: - BuildImagesAzure - ${{ if eq(parameters.includeUKI, true) }}: - BuildImagesUKI + - ${{ if eq(parameters.includeAzl4, true) }}: + - BuildImagesQemuGrubAzl4 variables: - group: servicing_testing_params @@ -234,6 +281,20 @@ stages: value: "trident-vm-grub-verity-testimage-$(System.DefinitionId)" jobs: + - ${{ if eq(parameters.includeAzl4, true) }}: + - ${{ each batch in parameters.workerBatches }}: + - template: testing-template.yml + parameters: + updateIterationCount: ${{ parameters.updateIterationCount }} + rollbackTesting: ${{ parameters.rollbackTesting }} + workers: ${{ batch.size }} + batchId: ${{ batch.id }} + updateCheckTimeoutInMinutes: ${{ parameters.updateCheckTimeoutInMinutes }} + verboseLogging: ${{ parameters.verboseLogging }} + platform: qemu + flavor: grubazl4 + testSecureBoot: ${{ parameters.testSecureBoot }} + - ${{ if eq(parameters.includeQemu, true) }}: - ${{ each batch in parameters.workerBatches }}: - template: testing-template.yml diff --git a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml index d6d1587aca..520966ed7c 100644 --- a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml +++ b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml @@ -63,6 +63,7 @@ parameters: values: - "2.0" - "3.0" + - "4.0-preview" default: "3.0" - name: baseimgTypes @@ -134,21 +135,6 @@ steps: cp $(Build.ArtifactStagingDirectory)/ssh/id_rsa* ${{ parameters.tridentSourceDirectory }}/artifacts/ displayName: Copy SSH Keys - # Stage the SSH public key into the test image tree so the builder bakes it - # into the image. The Makefile previously did this via the files/id_rsa.pub - # prerequisite; those explicit image targets were removed, so the pipeline - # now stages the key before building. Superseded by the shared - # prepare-testimage-requirements template. - - ${{ if eq(parameters.useStagedSshKeys, true) }}: - - bash: | - set -eux - - SRC="${{ parameters.tridentSourceDirectory }}/artifacts/id_rsa.pub" - DEST="${{ parameters.tridentSourceDirectory }}/tests/images/trident-vm-testimage/base/files" - mkdir -p "$DEST" - cp "$SRC" "$DEST/" - displayName: Stage SSH public key into testimage tree - - script: | echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING testimages.py TO BUILD TEST IMAGES." cat /etc/os-release @@ -227,60 +213,32 @@ steps: displayName: "Download Base Image - ${{ baseimgType }}" - ${{ else }}: - - template: common/base-images-download-template.yaml@platform-pipelines - parameters: - buildType: ${{ parameters.baseimgBuildType }} - baseImageType: ${{ baseimgType }} - imageVersion: ${{ parameters.baseimgVersion }} - azureLinuxVersion: ${{ parameters.baseimgAzureLinuxVersion }} - - # For dev builds, the RPMs are not necessarily published, so download them here. The - # parameter is set based on a runtime variable, so we cannot use ${{ if }}, so the - # template is always run and contains logic to skip if not needed. - - template: common/rpms-download-template.yaml@platform-pipelines + - ${{ if ne(parameters.baseimgAzureLinuxVersion, '4.0-preview') }}: + - template: common/base-images-download-template.yaml@platform-pipelines + parameters: + buildType: ${{ parameters.baseimgBuildType }} + baseImageType: ${{ baseimgType }} + imageVersion: ${{ parameters.baseimgVersion }} + azureLinuxVersion: ${{ parameters.baseimgAzureLinuxVersion }} + # For dev builds, the RPMs are not necessarily published, so download them here. The + # parameter is set based on a runtime variable, so we cannot use ${{ if }}, so the + # template is always run and contains logic to skip if not needed. + - template: common/rpms-download-template.yaml@platform-pipelines + parameters: + rpmsVersion: ${{ parameters.rpmsVersion }} + azureLinuxVersion: ${{ parameters.baseimgAzureLinuxVersion }} + runtimeBuildType: ${{ parameters.baseimgBuildType }} + + - ${{ else }}: + - template: ../common_tasks/download-azl4-base-vhdx.yml + parameters: + tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} + + - template: ../common_tasks/prepare-testimage-requirements.yml parameters: - rpmsVersion: ${{ parameters.rpmsVersion }} + tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} azureLinuxVersion: ${{ parameters.baseimgAzureLinuxVersion }} - runtimeBuildType: ${{ parameters.baseimgBuildType }} - - - bash: | - set -ex - - base_dir='${{ parameters.tridentSourceDirectory }}/artifacts' - - mkdir -p $base_dir - # Check if there are any .vhdx files in the images directory and move them - if ls "$(Build.ArtifactStagingDirectory)/images/" | grep -q ".*\.vhdx$"; then - mv $(Build.ArtifactStagingDirectory)/images/*.vhdx $base_dir/ - rm -rf $(Build.ArtifactStagingDirectory)/images - else - echo "No base image found" - exit 1 - fi - - if ls "$(Build.ArtifactStagingDirectory)/rpms/" | grep -q "rpms.tar.gz"; then - mkdir -p $base_dir/rpm-overrides - tar -xvf $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz \ - --strip-components=2 \ - -C $base_dir/rpm-overrides - rm $(Build.ArtifactStagingDirectory)/rpms/rpms.tar.gz - fi - - DISTRO=azl3 - if [[ "${{ parameters.baseimgAzureLinuxVersion }}" == "4.0-preview" ]]; then - DISTRO=azl4 - fi - - find $(Build.ArtifactStagingDirectory) - if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then - rpm_dir='${{ parameters.tridentSourceDirectory }}/bin/RPMS' - mkdir -p $rpm_dir - find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} $rpm_dir/ \; - rm -rf "$(Build.ArtifactStagingDirectory)/trident" - fi - - workingDirectory: ${{ parameters.tridentSourceDirectory }} - displayName: "Prepare and move requirements" + stageSshKeys: ${{ parameters.useStagedSshKeys }} - script: | # Meta From 7a58dfe7f213b47f157ad53257e75e13be6488a1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 17:00:12 +0000 Subject: [PATCH 2/7] switch azl_qemu_guest blob account; download cached netplan RPMs --- .../common_tasks/download-azl4-base-vhdx.yml | 72 +++++++++---------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml index 430cee0c30..375f2f9db6 100644 --- a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml +++ b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml @@ -2,52 +2,48 @@ parameters: - name: tridentSourceDirectory type: string - # The AZL4 base VHDX is sourced from the Azure Linux preview gallery - # backing storage account. The pipeline service connection must have - # `Storage Blob Data Reader` on this account. - # See tests/images/SERVICE-CONNECTION-RUNBOOK.md. - - name: blobStorageAccount - type: string - default: "azlpubdev2mruiyvi" - - - name: blobContainer - type: string - default: "images-dev" - - - name: blobSubscription - type: string - # Subscription where the storage account lives. The service connection - # default subscription may differ, so set context before download. - default: "e4ab81f8-030f-4593-a8f2-3ea2c7630a19" - - - name: blobServiceConnection - type: string - # NB: this must be a service connection that exists in the ADO project. - # Created manually by trident infra team. - default: "trident-azl4-blob-reader" - steps: - # Download the AZL4 base VHDX from the preview gallery backing storage. - # Authenticates via the federated identity attached to the service - # connection; no storage keys handled here. - # - # The service connection default subscription (Polar_ImageTools_Staging) - # differs from the storage account subscription (ControlTower_Test), so - # switch context so blob resolution targets the right account. + # Download a preview AZL4 base VHDX from storage account. - task: AzureCLI@2 displayName: "Download AZL4 base VHDX from blob" inputs: - azureSubscription: ${{ parameters.blobServiceConnection }} + azureSubscription: azlinuxbmpstaging-storage-account-read scriptType: bash scriptLocation: inlineScript workingDirectory: ${{ parameters.tridentSourceDirectory }} inlineScript: | set -euxo pipefail - az account set --subscription "${{ parameters.blobSubscription }}" - python3 ./tests/images/testimages.py download-image azl4_qemu_guest \ - --blob-storage-account "${{ parameters.blobStorageAccount }}" \ - --blob-container "${{ parameters.blobContainer }}" - ls -la artifacts/azl4_qemu_guest.vhdx + mkdir -p $(Build.ArtifactStagingDirectory)/images - mv artifacts/azl4_qemu_guest.vhdx $(Build.ArtifactStagingDirectory)/images/ + az storage blob download \ + --max-connections 10 \ + --auth-mode login \ + --account-name azlinuxbmpstaging \ + --container-name azl4-qemu-guest \ + --name azl4_qemu_guest.vhdx \ + --file $(Build.ArtifactStagingDirectory)/images/azl4_qemu_guest.vhdx ls -la $(Build.ArtifactStagingDirectory)/images/azl4_qemu_guest.vhdx + + # AZL4 netplan (1.1.2) is incompatible with systemd (258.4). 1.1.2 is the current + # release of netplan, but an unreleased refactor in main fixes the problem. Our + # tests depend on netplan, so workaround this for now with cached RPMs built from + # netplan unreleased main. + - task: AzureCLI@2 + displayName: "Download preview of unreleased netplan (branch=main) RPMs from blob" + inputs: + azureSubscription: azlinuxbmpstaging-storage-account-read + scriptType: bash + scriptLocation: inlineScript + workingDirectory: ${{ parameters.tridentSourceDirectory }} + inlineScript: | + set -euxo pipefail + + mkdir -p ${{ parameters.tridentSourceDirectory }}/bin/RPMS + az storage blob download-batch \ + --max-connections 10 \ + --auth-mode login \ + --account-name azlinuxbmpstaging \ + --source netplan-main \ + --pattern "*.rpm" \ + --destination ${{ parameters.tridentSourceDirectory }}/bin/RPMS + ls -la ${{ parameters.tridentSourceDirectory }}/bin/RPMS From 27015c7b202a9f15bb6423390395cca490b4652a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 20:44:08 +0000 Subject: [PATCH 3/7] make override more generic --- .../stages/common_tasks/download-azl4-base-vhdx.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml index 375f2f9db6..c9d4845a8e 100644 --- a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml +++ b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml @@ -25,11 +25,10 @@ steps: ls -la $(Build.ArtifactStagingDirectory)/images/azl4_qemu_guest.vhdx # AZL4 netplan (1.1.2) is incompatible with systemd (258.4). 1.1.2 is the current - # release of netplan, but an unreleased refactor in main fixes the problem. Our - # tests depend on netplan, so workaround this for now with cached RPMs built from - # netplan unreleased main. + # release of netplan in AZL4, but netplan 1.2.1 fixes the problem. Our + # tests depend on netplan, so workaround this for now with cached RPMs. - task: AzureCLI@2 - displayName: "Download preview of unreleased netplan (branch=main) RPMs from blob" + displayName: "Download override AZL4 RPMs from blob" inputs: azureSubscription: azlinuxbmpstaging-storage-account-read scriptType: bash @@ -43,7 +42,7 @@ steps: --max-connections 10 \ --auth-mode login \ --account-name azlinuxbmpstaging \ - --source netplan-main \ + --source azl4-override-rpms \ --pattern "*.rpm" \ --destination ${{ parameters.tridentSourceDirectory }}/bin/RPMS ls -la ${{ parameters.tridentSourceDirectory }}/bin/RPMS From 7e162be65380f46cfe3c79b1446804d18d3747d6 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 21:25:44 +0000 Subject: [PATCH 4/7] pipelines: enable grubazl4 netplan runtime rollback testing and raise step timeout Remove the grubazl4 force-skip of netplan runtime testing now that the early-boot systemd generator deadlock is fixed upstream (azurelinux PR #17791), and raise the rollback test step timeout from 5 to 10 minutes so the longer grubazl4 rollback suite can complete. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/testing_rollback/testing-template.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/stages/testing_rollback/testing-template.yml b/.pipelines/templates/stages/testing_rollback/testing-template.yml index 0083d9b8bd..f197d36253 100644 --- a/.pipelines/templates/stages/testing_rollback/testing-template.yml +++ b/.pipelines/templates/stages/testing_rollback/testing-template.yml @@ -179,12 +179,7 @@ jobs: # Allow testing for non-extension scenarios STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --skip-extension-testing" fi - # Netplan runtime testing is skipped on grubazl4 (Azure Linux 4) to - # avoid an early-boot systemd generator deadlock during rollback. - # Fixed upstream by azurelinux PR #17791 - # (https://github.com/microsoft/azurelinux/pull/17791/). - # Force it on for grubazl4 regardless of the pipeline parameter. - if [ "${{ parameters.skipNetplanRuntimeTesting }}" == "true" ] || [ "${{ parameters.flavor }}" == "grubazl4" ]; then + if [ "${{ parameters.skipNetplanRuntimeTesting }}" == "true" ]; then # skip netplan runtime testing STORM_DYNAMIC_FLAGS="$STORM_DYNAMIC_FLAGS --skip-netplan-runtime-testing" fi @@ -204,7 +199,7 @@ jobs: --force-cleanup displayName: "Rollback test (${{ parameters.platform }})" workingDirectory: $(TRIDENT_SOURCE_DIR) - timeoutInMinutes: 5 + timeoutInMinutes: 10 - bash: | set -eux From 6a0bf0e465e2a8b089cc6304820cd2cd0d5cf419 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 21:30:42 +0000 Subject: [PATCH 5/7] storm/rollback: mark test dummy netplan device optional to avoid wait-online stall The rollback netplan runtime test injects a dummy netplan device. By default systemd-networkd-wait-online waits for every managed link to be routable; the dummy virtual interface never becomes routable, so on AZL4 (netplan generate/configure split) wait-online can block for its full 120s timeout. That delays network-online.target -> trident.service and the post-update commit (ab-update-finalized -> provisioned), inflating each rollback subtest by up to two minutes. Setting optional: true emits RequiredForOnline=no so wait-online ignores the test dummy. The real uplink (vmeth*/eth0) still gates online. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/rollback/tests/helper.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/storm/rollback/tests/helper.go b/tools/storm/rollback/tests/helper.go index f7c0caa5ba..2bf2b53c6a 100644 --- a/tools/storm/rollback/tests/helper.go +++ b/tools/storm/rollback/tests/helper.go @@ -519,6 +519,12 @@ func (u *UpdateTest) createNetplanHostConfigSection() (map[string]interface{}, e if u.NetplanVersion > 0 { dummyDevices[fmt.Sprintf("dummy%d", u.NetplanVersion)] = map[string]interface{}{ "addresses": []string{fmt.Sprintf("192.168.%d.123/24", 100+u.NetplanVersion)}, + // Mark the test dummy interface optional so it emits + // RequiredForOnline=no. Otherwise systemd-networkd-wait-online + // blocks on this never-routable virtual link for its full 120s + // timeout, delaying network-online.target -> trident.service and + // the post-update commit (ab-update-finalized -> provisioned). + "optional": true, } } return map[string]interface{}{ From 5b34e5d85b4bfc48037649371f3dd17e90173719 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 21:37:58 +0000 Subject: [PATCH 6/7] pipelines: set stageSshKeys false in build-image-template Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/stages/build_image/build-image-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 8a250bf355..f7b1c9ecf8 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -127,7 +127,7 @@ steps: parameters: tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} azureLinuxVersion: ${{ parameters.azureLinuxVersion }} - stageSshKeys: ${{ eq(parameters.azureLinuxVersion, '4.0-preview') }} + stageSshKeys: false - bash: | set -ex From 52d938ec4a90c63e770b630f136351f438fcca62 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 27 Jun 2026 01:38:02 +0000 Subject: [PATCH 7/7] pipelines: address azl4 testimage review feedback - SSH staging: stage only id_rsa.pub into the testimage tree; do not copy or chmod the private key, which must not be baked into images. - download-azl4-base-vhdx: download OS override RPMs into the dedicated artifacts/rpm-overrides channel (always included by the builder) instead of bin/RPMS (Trident-only, gated on requires_trident). - prepare-testimage-requirements: harden the prepare/move block with set -euxo pipefail. - build-image-template: fail fast for AZL4 preview if the resolved base image type is not azl4_qemu_guest (the only type download-azl4 fetches). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build_image/build-image-template.yml | 10 ++++++++++ .../stages/common_tasks/download-azl4-base-vhdx.yml | 6 +++--- .../common_tasks/prepare-testimage-requirements.yml | 9 ++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index f7b1c9ecf8..8e9897c495 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -119,6 +119,16 @@ steps: azureLinuxVersion: ${{ parameters.azureLinuxVersion }} runtimeBuildType: ${{ parameters.baseimgBuildType }} - ${{ else }}: + # download-azl4-base-vhdx.yml only fetches the azl4_qemu_guest VHDX, + # so fail fast if the image resolves to a different base image type. + - bash: | + set -euxo pipefail + if [[ "$(baseImageType)" != "azl4_qemu_guest" ]]; then + echo "ERROR: AZL4 preview only supports base image type 'azl4_qemu_guest', but '$(baseImageType)' was resolved for image '${{ parameters.imageName }}'." + exit 1 + fi + displayName: "Validate AZL4 base image type" + workingDirectory: ${{ parameters.tridentSourceDirectory }} - template: ../common_tasks/download-azl4-base-vhdx.yml parameters: tridentSourceDirectory: ${{ parameters.tridentSourceDirectory }} diff --git a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml index c9d4845a8e..a420f89298 100644 --- a/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml +++ b/.pipelines/templates/stages/common_tasks/download-azl4-base-vhdx.yml @@ -37,12 +37,12 @@ steps: inlineScript: | set -euxo pipefail - mkdir -p ${{ parameters.tridentSourceDirectory }}/bin/RPMS + mkdir -p ${{ parameters.tridentSourceDirectory }}/artifacts/rpm-overrides az storage blob download-batch \ --max-connections 10 \ --auth-mode login \ --account-name azlinuxbmpstaging \ --source azl4-override-rpms \ --pattern "*.rpm" \ - --destination ${{ parameters.tridentSourceDirectory }}/bin/RPMS - ls -la ${{ parameters.tridentSourceDirectory }}/bin/RPMS + --destination ${{ parameters.tridentSourceDirectory }}/artifacts/rpm-overrides + ls -la ${{ parameters.tridentSourceDirectory }}/artifacts/rpm-overrides diff --git a/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml b/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml index 72b839131e..909cb72678 100644 --- a/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml +++ b/.pipelines/templates/stages/common_tasks/prepare-testimage-requirements.yml @@ -27,15 +27,18 @@ steps: exit 1 fi + # Stage only the public key. The image configs reference only + # files/id_rsa.pub (injected as an authorized key); the private + # key is used host-side by the test runner and must not be baked + # into the image. mkdir -p "$SSH_DEST" - cp "$SSH_SRC/id_rsa"* "$SSH_DEST/" - chmod 600 "$SSH_DEST/id_rsa" + cp "$SSH_SRC/id_rsa.pub" "$SSH_DEST/" chmod 644 "$SSH_DEST/id_rsa.pub" displayName: "Stage SSH keys into testimage tree" workingDirectory: ${{ parameters.tridentSourceDirectory }} - bash: | - set -ex + set -euxo pipefail # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx) mkdir -p artifacts