From 0601b442af9a3e15b8c6d5b967fb006fe3419e3d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 24 Jun 2026 23:46:35 +0000 Subject: [PATCH 01/19] azl4: e2e test configurations Add the Azure Linux 4 end-to-end test configurations (base-azl4 and rollback-azl4 trident configs + test selection) and the base_test.py plumbing that drives them. Stacked on the azl4 pipeline PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/e2e_tests/base_test.py | 3 + .../base-azl4/test-selection.yaml | 5 ++ .../base-azl4/trident-config.yaml | 74 +++++++++++++++++ .../rollback-azl4/test-selection.yaml | 3 + .../rollback-azl4/trident-config.yaml | 80 +++++++++++++++++++ 5 files changed, 165 insertions(+) create mode 100644 tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml create mode 100644 tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml create mode 100644 tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml create mode 100644 tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml diff --git a/tests/e2e_tests/base_test.py b/tests/e2e_tests/base_test.py index 3e60948054..884e2dcec1 100644 --- a/tests/e2e_tests/base_test.py +++ b/tests/e2e_tests/base_test.py @@ -421,6 +421,9 @@ def test_users(connection, hostConfiguration): expected_users = list() expected_groups = dict() + if "os" not in hostConfiguration or "users" not in hostConfiguration.get("os", {}): + pytest.skip("No os.users in trident config (user baked into image by MIC)") + for user_info in hostConfiguration["os"]["users"]: expected_users.append(user_info["name"]) if "groups" in user_info: diff --git a/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml new file mode 100644 index 0000000000..1789997bd2 --- /dev/null +++ b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml @@ -0,0 +1,5 @@ +compatible: + - base-azl4 + # Reuse the same pytest assertions as the AZL3 `base` scenario where + # appropriate. Add this scenario explicitly to test markers as we wire + # up pytest coverage. diff --git a/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml new file mode 100644 index 0000000000..c918e15151 --- /dev/null +++ b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml @@ -0,0 +1,74 @@ +image: + url: http://NETLAUNCH_HOST_ADDRESS/files/regular.cosi + sha384: ignored +# AZL4 does not ship grub2-efi-binary-noprefix. Trident handles this +# automatically — no disableGrubNoprefixCheck override needed. +storage: + disks: + - id: os + device: /dev/disk/by-path/pci-0000:00:1f.2-ata-2 + partitionTableType: gpt + partitions: + - id: root-a + type: root + size: 8G + - id: root-b + type: root + size: 8G + - id: esp + type: esp + size: 1G + - id: swap + type: swap + size: 2G + - id: trident + type: linux-generic + size: 1G + - id: disk2 + device: /dev/disk/by-path/pci-0000:00:1f.2-ata-3 + partitionTableType: gpt + partitions: [] + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + filesystems: + - deviceId: trident + source: new + mountPoint: /var/lib/trident + - deviceId: esp + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: root + mountPoint: / + swap: + - swap +# /home partition omitted: the COSI bakes a user home directory onto +# root via MIC os.users. Trident's newroot mount rejects non-empty +# mount points, so a separate /home partition conflicts with the +# pre-existing /home/. AZL3 avoids this by only testing /home +# in container mode. Container mode support for AZL4 is tracked as +# a follow-up. +scripts: + postConfigure: + - name: testing-privilege + runOn: + - clean-install + - ab-update + content: echo 'testing-user ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/testing-user +os: + selinux: + mode: disabled + netplan: + version: 2 + ethernets: + vmeths: + match: + name: eth* + dhcp4: true + users: + - name: testing-user + sshPublicKeys: [] + sshMode: key-only \ No newline at end of file diff --git a/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml new file mode 100644 index 0000000000..cbfa81bbe7 --- /dev/null +++ b/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml @@ -0,0 +1,3 @@ +compatible: + - rollback + - rollback-azl4 diff --git a/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml b/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml new file mode 100644 index 0000000000..67a2c574a6 --- /dev/null +++ b/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml @@ -0,0 +1,80 @@ +image: + url: http://NETLAUNCH_HOST_ADDRESS/files/regular.cosi + sha384: ignored +storage: + disks: + - id: os + device: /dev/disk/by-path/pci-0000:00:1f.2-ata-2 + partitionTableType: gpt + partitions: + - id: root-a + type: root + size: 8G + - id: root-b + type: root + size: 8G + - id: esp + type: esp + size: 1G + - id: trident + type: linux-generic + size: 1G + - id: disk2 + device: /dev/disk/by-path/pci-0000:00:1f.2-ata-3 + partitionTableType: gpt + partitions: [] + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + filesystems: + - deviceId: trident + source: new + mountPoint: /var/lib/trident + - deviceId: esp + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: root + mountPoint: / +os: + additionalFiles: + - destination: /var/lib/trident/local-health-check-file.sh + content: | + echo 'This is a local health check script.' + exit 0 +health: + checks: + - name: invoke-rollback-from-local-script + runOn: + - clean-install + path: /var/lib/trident/local-health-check-file.sh + - name: invoke-rollback-from-script + runOn: + - clean-install + content: | + exit 1 + - name: install-failure-systemd-check + runOn: + - clean-install + systemdServices: + - non-existent-service1.service + - non-existent-service2.service + timeoutSeconds: 15 +# AZL4 variant of the AZL3 `health-checks-install/` scenario. +# - No `users`/`selinux`/`netplan` — these are baked into the test image +# at MIC build time. +# - `os.additionalFiles` is used because health.checks references +# `path: /var/lib/trident/local-health-check-file.sh`, which needs to +# be on the target filesystem. +# +# Health-check failure expectations (asserted by tests/e2e_tests/rollback_test.py): +# - State transitions to `not-provisioned` (clean-install has no slot to +# roll back to; the install just fails). +# - `/var/lib/trident/trident-health-check-failure-*.log` is created. +# - The log contains: +# * `"Failed health check(s)"` +# * `"Script 'invoke-rollback-from-script' failed"` +# * `"Unit non-existent-service1.service could not be found"` +# * `"Unit non-existent-service2.service could not be found"` From 205c4d4c7743a1ddc926d067ce37dfa100917eed Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 17:29:42 +0000 Subject: [PATCH 02/19] enable e2e tests --- .pipelines/templates/e2e-template.yml | 17 +++ .../stages/common_tasks/push-to-acr.yml | 1 + .../stages/common_tasks/remove-from-acr.yml | 1 + .../testing_common/download-test-images.yml | 1 + .../stages/testing_common/trident-prep.yml | 1 + .../stages/testing_vm/netlaunch-testing.yml | 133 +++++++++++++----- tests/images/testimages.py | 9 ++ .../trident-testimage/base/baseimg-azl4.yaml | 115 +++++++++++++++ .../base/files/hostname-shim.sh | 20 +++ .../base/scripts/ssh-move-host-keys-azl4.sh | 13 ++ .../base/scripts/strip-selinux-xattrs.sh | 85 +++++++++++ 11 files changed, 357 insertions(+), 39 deletions(-) create mode 100644 tests/images/trident-testimage/base/baseimg-azl4.yaml create mode 100644 tests/images/trident-testimage/base/files/hostname-shim.sh create mode 100755 tests/images/trident-testimage/base/scripts/ssh-move-host-keys-azl4.sh create mode 100644 tests/images/trident-testimage/base/scripts/strip-selinux-xattrs.sh diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index bf5e827c77..8fe70156cd 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -181,6 +181,15 @@ stages: micVersion: ${{ parameters.micVersion }} dependsOnStage: ${{ parameters.baseImageArtifactStage }} + # Build Trident test image (regular) for AZL4 + - template: stages/build_image/build-image.yml + parameters: + imageName: trident-azl4-testimage + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} + azureLinuxVersion: "4.0-preview" + # Build Trident test image (container) - template: stages/build_image/build-image.yml parameters: @@ -409,6 +418,14 @@ stages: buildPurpose: "pullrequest" runtimeEnv: "container" + # VM Testing (host, pullrequest) for AZL4 + - template: stages/testing_vm/netlaunch-testing.yml + parameters: + buildPurpose: "pullrequest" + runtimeEnv: "host" + distro: azl4 + stageSuffix: "_azl4" + # Run direct streaming tests for amd64 with streaming images - template: direct-streaming-test.yml parameters: diff --git a/.pipelines/templates/stages/common_tasks/push-to-acr.yml b/.pipelines/templates/stages/common_tasks/push-to-acr.yml index b6d16d075d..cc63b2b5cf 100644 --- a/.pipelines/templates/stages/common_tasks/push-to-acr.yml +++ b/.pipelines/templates/stages/common_tasks/push-to-acr.yml @@ -17,6 +17,7 @@ parameters: type: string values: - trident-testimage + - trident-azl4-testimage - trident-container-testimage - name: "config" diff --git a/.pipelines/templates/stages/common_tasks/remove-from-acr.yml b/.pipelines/templates/stages/common_tasks/remove-from-acr.yml index 0c4dcb792c..a061ddb12f 100644 --- a/.pipelines/templates/stages/common_tasks/remove-from-acr.yml +++ b/.pipelines/templates/stages/common_tasks/remove-from-acr.yml @@ -17,6 +17,7 @@ parameters: type: string values: - trident-testimage + - trident-azl4-testimage - trident-container-testimage - name: "config" diff --git a/.pipelines/templates/stages/testing_common/download-test-images.yml b/.pipelines/templates/stages/testing_common/download-test-images.yml index 141c3cb4ed..3c4a431574 100644 --- a/.pipelines/templates/stages/testing_common/download-test-images.yml +++ b/.pipelines/templates/stages/testing_common/download-test-images.yml @@ -19,6 +19,7 @@ parameters: values: - trident-container-testimage - trident-testimage + - trident-azl4-testimage - azurelinux-direct-streaming-testimage-amd64 - azurelinux-direct-streaming-testimage-arm64 - ubuntu-direct-streaming-testimage-2204-amd64 diff --git a/.pipelines/templates/stages/testing_common/trident-prep.yml b/.pipelines/templates/stages/testing_common/trident-prep.yml index d3276dc5c9..4d39668ae6 100644 --- a/.pipelines/templates/stages/testing_common/trident-prep.yml +++ b/.pipelines/templates/stages/testing_common/trident-prep.yml @@ -19,6 +19,7 @@ parameters: default: trident-testimage values: - trident-testimage + - trident-azl4-testimage - trident-container-testimage - name: config diff --git a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml index cd7b4187a7..26cf2b80b3 100644 --- a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml +++ b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml @@ -39,9 +39,20 @@ parameters: - maritimus-dev-acr-write-umi - trident-dev-acr-write-umi-ECF + - name: distro + type: string + default: azl3 + values: + - azl3 + - azl4 + + - name: stageSuffix + type: string + default: "" + stages: - - stage: DefineTests_VM_${{ parameters.runtimeEnv }} - displayName: Test List for VM:${{ parameters.runtimeEnv }} + - stage: DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} + displayName: Test List for VM:${{ parameters.runtimeEnv }} (${{ parameters.distro }}) jobs: - template: ../testing_common/get-tests.yml parameters: @@ -50,26 +61,43 @@ stages: runtimeEnv: ${{ parameters.runtimeEnv }} testSecureBoot: ${{ parameters.testSecureBoot }} - - stage: DeploymentTesting_${{ parameters.runtimeEnv }} + - stage: DeploymentTesting_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing dependsOn: - - DefineTests_VM_${{ parameters.runtimeEnv }} + - DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} - ${{ if eq(parameters.testingRun, true) }}: - DownloadTestingElements - ${{ else }}: - BuildingTools - - ${{ if eq(parameters.runtimeEnv, 'container') }}: - - BuildTridentContainerImage - - TridentTestImg_trident_container_installer - - TridentTestImg_trident_container_testimage - - TridentTestImg_trident_container_verity_testimage - - TridentTestImg_trident_container_usrverity_testimage - - ${{ else }}: - - TridentTestImg_trident_split_installer - - TridentTestImg_trident_installer - - TridentTestImg_trident_testimage - - TridentTestImg_trident_verity_testimage - - TridentTestImg_trident_usrverity_testimage + - ${{ if eq(parameters.distro, 'azl3') }}: + - ${{ if eq(parameters.runtimeEnv, 'container') }}: + - BuildTridentContainerImage + - TridentTestImg_trident_container_installer + - TridentTestImg_trident_container_testimage + - TridentTestImg_trident_container_verity_testimage + - TridentTestImg_trident_container_usrverity_testimage + - ${{ else }}: + - TridentTestImg_trident_split_installer + - TridentTestImg_trident_installer + - TridentTestImg_trident_testimage + - TridentTestImg_trident_verity_testimage + - TridentTestImg_trident_usrverity_testimage + - ${{ if eq(parameters.distro, 'azl4') }}: + - ${{ if eq(parameters.runtimeEnv, 'container') }}: + # TODO: the following are placeholders until container supported for AZL4 + - BuildTridentContainerImage + - TridentTestImg_trident_container_installer + - TridentTestImg_trident_container_testimage + - TridentTestImg_trident_container_verity_testimage + - TridentTestImg_trident_container_usrverity_testimage + - ${{ else }}: + - TridentTestImg_trident_installer + - TridentTestImg_trident_azl4_testimage + # TODO: the following are placeholders until the verity + # and usrverity test images are built for AZL4 + - TridentTestImg_trident_split_installer + - TridentTestImg_trident_verity_testimage + - TridentTestImg_trident_usrverity_testimage jobs: - job: Testing @@ -80,7 +108,7 @@ stages: hostArchitecture: amd64 strategy: - matrix: $[ stageDependencies.DefineTests_VM_${{ parameters.runtimeEnv }}.DefineTests.outputs['setConfigurations.matrixConfigurations'] ] + matrix: $[ stageDependencies.DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }}.DefineTests.outputs['setConfigurations.matrixConfigurations'] ] variables: # Sourced from the matrix @@ -90,28 +118,55 @@ stages: - name: tridentConfigPath value: tests/e2e_tests/trident_configurations/$(tridentConfigurationName) - - ${{ if eq(parameters.runtimeEnv, 'container') }}: - - name: installerISOName - value: trident-container-installer - - name: testImageName - value: trident-container-testimage - - name: verityTestImageName - value: trident-container-verity-testimage - - name: usrVerityTestImageName - value: trident-container-usrverity-testimage - - name: downloadTridentContainer - value: true - - ${{ else }}: - - name: installerISOName - value: trident-installer - - name: testImageName - value: trident-testimage - - name: verityTestImageName - value: trident-verity-testimage - - name: usrVerityTestImageName - value: trident-usrverity-testimage - - name: downloadTridentContainer - value: false + - ${{ if eq(parameters.distro, 'azl3') }}: + - ${{ if eq(parameters.runtimeEnv, 'container') }}: + - name: installerISOName + value: trident-container-installer + - name: testImageName + value: trident-container-testimage + - name: verityTestImageName + value: trident-container-verity-testimage + - name: usrVerityTestImageName + value: trident-container-usrverity-testimage + - name: downloadTridentContainer + value: true + - ${{ else }}: + - name: installerISOName + value: trident-installer + - name: testImageName + value: trident-testimage + - name: verityTestImageName + value: trident-verity-testimage + - name: usrVerityTestImageName + value: trident-usrverity-testimage + - name: downloadTridentContainer + value: false + - ${{ if eq(parameters.distro, 'azl4') }}: + - ${{ if eq(parameters.runtimeEnv, 'container') }}: + # TODO: the following variables are placeholders until container supported for AZL4 + - name: installerISOName + value: trident-container-installer + - name: testImageName + value: trident-container-testimage + - name: verityTestImageName + value: trident-container-verity-testimage + - name: usrVerityTestImageName + value: trident-container-usrverity-testimage + - name: downloadTridentContainer + value: true + - ${{ else }}: + - name: installerISOName + value: trident-installer + - name: testImageName + value: trident-azl4-testimage + # TODO: the following variables are placeholders until the verity + # and usrverity test images are built for AZL4 + - name: verityTestImageName + value: trident-verity-testimage + - name: usrVerityTestImageName + value: trident-usrverity-testimage + - name: downloadTridentContainer + value: false - name: ob_outputDirectory value: /tmp/deployment_logs diff --git a/tests/images/testimages.py b/tests/images/testimages.py index 9a0683f76f..89c67403f9 100755 --- a/tests/images/testimages.py +++ b/tests/images/testimages.py @@ -196,6 +196,15 @@ ssh_key="files/id_rsa.pub", architecture=SystemArchitecture.ARM64, ), + # Test images (azl4) + ImageConfig( + "trident-azl4-testimage", + config="trident-testimage", + base_image=BaseImage.AZL4_QEMU_GUEST, + output_and_config={ + OutputFormat.COSI: "base/baseimg-azl4.yaml", + }, + ), # VM test images (azl4) ImageConfig( "trident-vm-grub-azl4-testimage", diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml new file mode 100644 index 0000000000..6bf1c9502e --- /dev/null +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -0,0 +1,115 @@ +storage: + bootType: efi + + disks: + - partitionTableType: gpt + maxSize: 4G + partitions: + - id: esp + type: esp + size: 8M + + - id: rootfs + size: grow + + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + + - deviceId: rootfs + type: ext4 + mountPoint: + path: / + +os: + bootloader: + resetType: hard-reset + hostname: trident-testimg + + kernelCommandLine: + # Replicates BM base image settings, that would otherwise be lost + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + + packages: + install: + - curl + # AZL4: dnf5 (no `dnf` package on AZL4) + - dnf5 + # ^^^ + - efibootmgr + # AZL4: grub2-efi-x64 + grub2-efi-x64-modules (no `-noprefix` package) + - grub2-efi-x64 + - grub2-efi-x64-modules + - grub2-tools + - grub2-tools-efi + # ^^^ + - iproute + # AZL4: iptables-nft (no plain `iptables` meta on AZL4 base) + - iptables-nft + # ^^^ + - lsof + - mdadm + - netplan + - netplan-default-backend-networkd + - openssh-server + - tpm2-tools + # AZL4: shim added (AZL4 ships a real signed shim chain) + - shim + # WHY? maybe baseimage is missing them? + - sudo + - systemd-networkd + - systemd-resolved + # AZL4 systemd (258.4) moves systemd-dissect and mount.ddi from the + # main systemd package into the systemd-container subpackage. mount.ddi + # is needed for sysext/sysconf + - systemd-sysext + - systemd-confext + - systemd-dissect + # ^^^ + - trident-service + - vim + - audit + # Packages required for functional tests, which currently use this image: + - device-mapper + - dosfstools + - lvm2 + - veritysetup + # Optional dependencies for NTFS + - ntfs-3g + - ntfsprogs + + services: + enable: + - sshd + - trident + - tridentd.socket + + additionalFiles: + # AZL4 lacks a /usr/bin/hostname binary; the pytest framework smoke- + # tests SSH with `hostname`, so we ship a tiny shim. + - source: files/hostname-shim.sh + destination: /usr/local/bin/hostname + permissions: "755" + # For tests, override the public trident.service behavior and use + # grpc-client commit. + - source: files/use-grpc-client-commit.conf + destination: /etc/systemd/system/trident.service.d/override.conf + +scripts: + postCustomization: + - path: post-install.sh + - path: scripts/ssh-move-host-keys-azl4.sh + # Strip SELinux labels from all files. See the script header for the + # full rationale, but in short: the AZL4 base VHDX bakes in AZL4 + # SELinux contexts that Trident-from-AZL3-MOS can't preserve during + # install (MOS's loaded policy rejects unknown contexts via + # setxattr). Stripping at cosi build time sidesteps the cascade. + # Must run LAST so other postCustomization scripts don't re-label. + - path: scripts/strip-selinux-xattrs.sh \ No newline at end of file diff --git a/tests/images/trident-testimage/base/files/hostname-shim.sh b/tests/images/trident-testimage/base/files/hostname-shim.sh new file mode 100644 index 0000000000..b12b3807c9 --- /dev/null +++ b/tests/images/trident-testimage/base/files/hostname-shim.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# AZL4 doesn't ship a `hostname` binary in `coreutils` (Fedora moved it to +# its own package which AZL4 hasn't picked up yet). The pytest E2E +# framework uses `hostname` as a smoke test of the SSH session in +# tests/e2e_tests/conftest.py, so without this shim every test errors out +# at fixture setup. +# +# Tiny POSIX-only replacement that reads /etc/hostname, plus a passthrough +# for `hostname -s` and `hostname -f` for completeness. +case "$1" in + -s|--short) + cat /etc/hostname | cut -d. -f1 + ;; + -f|--fqdn|"") + cat /etc/hostname + ;; + *) + cat /etc/hostname + ;; +esac diff --git a/tests/images/trident-testimage/base/scripts/ssh-move-host-keys-azl4.sh b/tests/images/trident-testimage/base/scripts/ssh-move-host-keys-azl4.sh new file mode 100755 index 0000000000..ede3fdbaa2 --- /dev/null +++ b/tests/images/trident-testimage/base/scripts/ssh-move-host-keys-azl4.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# AZL4-compatible variant of ssh-move-host-keys.sh. +# +# AZL3 sshd reads the main /etc/ssh/sshd_config and we appended HostKey +# lines to it. AZL4 sshd 10.0+ supports drop-ins under /etc/ssh/sshd_config.d/ +# which is the cleaner approach. +SSH_VAR_DIR="/var/srv/etc/ssh" +mkdir -p /etc/ssh/sshd_config.d +cat > /etc/ssh/sshd_config.d/50-trident-host-keys.conf <&1 >/dev/null) || rc=$? && rc=${rc:-0} + if [ "$rc" -eq 0 ]; then + count=$((count + 1)) + elif echo "$err" | grep -qE "No such attribute|Operation not supported"; then + : # nothing to strip, expected for files without the xattr + else + fail_count=$((fail_count + 1)) + echo "setfattr failed on '$f': $err" >&2 + fi + rc=0 +done < <(find / \( -path /proc -o -path /sys -o -path /dev -o -path /run \) -prune \ + -o \( -type f -o -type d -o -type l \) -print0) + +echo "Stripped security.selinux from ${count} files/dirs" + +if [ "$fail_count" -gt 0 ]; then + echo "ERROR: setfattr failed (non-ENODATA) on ${fail_count} entries" >&2 + exit 1 +fi + +# Verify the strip actually took effect by scanning a representative set +# of paths (rootfs, /boot if present, /usr/lib/systemd, /etc). Any +# residual security.selinux means we missed something — fail loudly +# rather than warning, since the whole point of the script is to leave +# the image bare. +sentinel_dirs=( "/etc" "/usr/lib/systemd" "/usr/bin" ) +if [ -d /boot ]; then + sentinel_dirs+=( "/boot" ) +fi +for d in "${sentinel_dirs[@]}"; do + if getfattr -R -m security.selinux "$d" 2>/dev/null | grep -q security.selinux; then + echo "ERROR: security.selinux xattr still present under '$d'" >&2 + getfattr -R -m security.selinux "$d" 2>/dev/null | head -10 >&2 + exit 1 + fi +done From 6bb5964c4a6af98d940754c0c982ac265f444a8f Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 19:38:29 +0000 Subject: [PATCH 03/19] turn off secure boot for now; separate distro tests --- .pipelines/templates/e2e-template.yml | 1 + .../stages/testing_common/get-tests.yml | 7 ++++ .../stages/testing_vm/netlaunch-testing.yml | 1 + .../helpers/read_target_configurations.py | 32 +++++++++++++++++++ tests/e2e_tests/target-configurations.yaml | 6 ++++ .../base-azl4/test-selection.yaml | 2 +- .../rollback-azl4/test-selection.yaml | 2 +- .../trident-testimage/base/baseimg-azl4.yaml | 4 +-- 8 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 8fe70156cd..93990bd373 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -425,6 +425,7 @@ stages: runtimeEnv: "host" distro: azl4 stageSuffix: "_azl4" + testSecureBoot: false # Run direct streaming tests for amd64 with streaming images - template: direct-streaming-test.yml diff --git a/.pipelines/templates/stages/testing_common/get-tests.yml b/.pipelines/templates/stages/testing_common/get-tests.yml index 646711bfac..1ec15e40ec 100644 --- a/.pipelines/templates/stages/testing_common/get-tests.yml +++ b/.pipelines/templates/stages/testing_common/get-tests.yml @@ -31,6 +31,13 @@ parameters: type: boolean default: true + - name: distro + type: string + default: "azl3" + values: + - azl3 + - azl4 + jobs: - job: DefineTests diff --git a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml index 26cf2b80b3..5cc32f5d45 100644 --- a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml +++ b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml @@ -60,6 +60,7 @@ stages: deploymentEnvironment: virtualMachine runtimeEnv: ${{ parameters.runtimeEnv }} testSecureBoot: ${{ parameters.testSecureBoot }} + distro: ${{ parameters.distro }} - stage: DeploymentTesting_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing diff --git a/tests/e2e_tests/helpers/read_target_configurations.py b/tests/e2e_tests/helpers/read_target_configurations.py index bbeb9cca38..09f1d7d5f5 100755 --- a/tests/e2e_tests/helpers/read_target_configurations.py +++ b/tests/e2e_tests/helpers/read_target_configurations.py @@ -31,6 +31,14 @@ def main(): choices=["virtualMachine", "bareMetal"], help="Deployment environment that will be used.", ) + parser.add_argument( + "-d", + "--distro", + type=str, + default="azl3", + choices=["azl3", "azl4"], + help="Distro version that will be used.", + ) parser.add_argument( "-p", "--purpose", @@ -84,6 +92,30 @@ def main(): configurations = target_configurations[args.env][args.runtimeEnv][args.purpose] + azl4_compatible_configurations = [] + non_azl4_compatible_configurations = [] + for config in configurations: + configurations_path = Path(configurations_file).parent + config_path = ( + configurations_path + / "trident_configurations" + / config + / "test-selection.yaml" + ) + with open(config_path, "r") as config_file: + config_as_yaml = yaml.safe_load(config_file) + log.info(f"Read configuration: {config_as_yaml}.") + if config_as_yaml.get("compatible", []) and "azl4" in config_as_yaml.get( + "compatible", [] + ): + azl4_compatible_configurations.append(config) + else: + non_azl4_compatible_configurations.append(config) + if args.distro == "azl4": + configurations = azl4_compatible_configurations + else: + configurations = non_azl4_compatible_configurations + if args.skipEncryptionTests: log.info(f"Skipping encryption tests as per the argument --skipEncryptionTests") non_encryption_configurations = [] diff --git a/tests/e2e_tests/target-configurations.yaml b/tests/e2e_tests/target-configurations.yaml index 017c0eb7f2..a737487f5a 100644 --- a/tests/e2e_tests/target-configurations.yaml +++ b/tests/e2e_tests/target-configurations.yaml @@ -87,6 +87,7 @@ virtualMachine: host: daily: - base + - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -106,6 +107,7 @@ virtualMachine: - usr-verity-raid post_merge: - base + - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -125,6 +127,7 @@ virtualMachine: - usr-verity-raid pullrequest: - base + - base-azl4 - combined - extensions - health-checks-install @@ -133,12 +136,14 @@ virtualMachine: - raid-resync-small - rerun - root-verity + - rollback-azl4 - simple - split - usr-verity - usr-verity-raid validation: - base + - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -158,6 +163,7 @@ virtualMachine: - usr-verity-raid weekly: - base + - base-azl4 - combined - encrypted-partition - encrypted-raid diff --git a/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml index 1789997bd2..eb1a3aca31 100644 --- a/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml +++ b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml @@ -1,5 +1,5 @@ compatible: - - base-azl4 + - azl4 # Reuse the same pytest assertions as the AZL3 `base` scenario where # appropriate. Add this scenario explicitly to test markers as we wire # up pytest coverage. diff --git a/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml index cbfa81bbe7..2ea207a640 100644 --- a/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml +++ b/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml @@ -1,3 +1,3 @@ compatible: + - azl4 - rollback - - rollback-azl4 diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index 6bf1c9502e..fed5a7e3dd 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -3,11 +3,11 @@ storage: disks: - partitionTableType: gpt - maxSize: 4G + maxSize: 5G partitions: - id: esp type: esp - size: 8M + size: 16M - id: rootfs size: grow From ded2b08a5a67e60ceacea1b20f3aebc6b0a0d30c Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 20:42:31 +0000 Subject: [PATCH 04/19] try --- .pipelines/templates/stages/testing_common/get-tests.yml | 1 + .pipelines/templates/stages/testing_vm/netlaunch-testing.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/testing_common/get-tests.yml b/.pipelines/templates/stages/testing_common/get-tests.yml index 1ec15e40ec..eec16d7811 100644 --- a/.pipelines/templates/stages/testing_common/get-tests.yml +++ b/.pipelines/templates/stages/testing_common/get-tests.yml @@ -63,6 +63,7 @@ jobs: --env ${{ parameters.deploymentEnvironment }} \ --runtimeEnv ${{ parameters.runtimeEnv }} \ --purpose ${{ parameters.buildPurpose }} \ + --distro ${{ parameters.distro }} \ $SKIP_ENCRYPTION_TESTS_ARG \ --matrix-name matrixConfigurations name: setConfigurations diff --git a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml index 5cc32f5d45..e99e9e809c 100644 --- a/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml +++ b/.pipelines/templates/stages/testing_vm/netlaunch-testing.yml @@ -63,7 +63,7 @@ stages: distro: ${{ parameters.distro }} - stage: DeploymentTesting_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} - displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing + displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing (${{ parameters.distro }}) dependsOn: - DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }} - ${{ if eq(parameters.testingRun, true) }}: From da44cb7d0e1c5159fcf0a065598913a5dc4d69ac Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 21:07:02 +0000 Subject: [PATCH 05/19] try --- .../stages/testing_common/get-tests.yml | 7 ++- .../helpers/read_target_configurations.py | 32 ----------- .../e2e_tests/target-configurations-azl4.yaml | 54 +++++++++++++++++++ tests/e2e_tests/target-configurations.yaml | 6 --- 4 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 tests/e2e_tests/target-configurations-azl4.yaml diff --git a/.pipelines/templates/stages/testing_common/get-tests.yml b/.pipelines/templates/stages/testing_common/get-tests.yml index eec16d7811..c70e7bd960 100644 --- a/.pipelines/templates/stages/testing_common/get-tests.yml +++ b/.pipelines/templates/stages/testing_common/get-tests.yml @@ -58,12 +58,15 @@ jobs: if [ "${{ parameters.testSecureBoot }}" == "False" ]; then SKIP_ENCRYPTION_TESTS_ARG="--skipEncryptionTests" fi + CONFIGURATIONS_FILE=./tests/e2e_tests/target-configurations.yaml + if [ "${{ parameters.distro }}" == "azl4" ]; then + CONFIGURATIONS_FILE=./tests/e2e_tests/target-configurations-azl4.yaml + fi python3 ./tests/e2e_tests/helpers/read_target_configurations.py \ - --configurations ./tests/e2e_tests/target-configurations.yaml \ + --configurations $CONFIGURATIONS_FILE \ --env ${{ parameters.deploymentEnvironment }} \ --runtimeEnv ${{ parameters.runtimeEnv }} \ --purpose ${{ parameters.buildPurpose }} \ - --distro ${{ parameters.distro }} \ $SKIP_ENCRYPTION_TESTS_ARG \ --matrix-name matrixConfigurations name: setConfigurations diff --git a/tests/e2e_tests/helpers/read_target_configurations.py b/tests/e2e_tests/helpers/read_target_configurations.py index 09f1d7d5f5..bbeb9cca38 100755 --- a/tests/e2e_tests/helpers/read_target_configurations.py +++ b/tests/e2e_tests/helpers/read_target_configurations.py @@ -31,14 +31,6 @@ def main(): choices=["virtualMachine", "bareMetal"], help="Deployment environment that will be used.", ) - parser.add_argument( - "-d", - "--distro", - type=str, - default="azl3", - choices=["azl3", "azl4"], - help="Distro version that will be used.", - ) parser.add_argument( "-p", "--purpose", @@ -92,30 +84,6 @@ def main(): configurations = target_configurations[args.env][args.runtimeEnv][args.purpose] - azl4_compatible_configurations = [] - non_azl4_compatible_configurations = [] - for config in configurations: - configurations_path = Path(configurations_file).parent - config_path = ( - configurations_path - / "trident_configurations" - / config - / "test-selection.yaml" - ) - with open(config_path, "r") as config_file: - config_as_yaml = yaml.safe_load(config_file) - log.info(f"Read configuration: {config_as_yaml}.") - if config_as_yaml.get("compatible", []) and "azl4" in config_as_yaml.get( - "compatible", [] - ): - azl4_compatible_configurations.append(config) - else: - non_azl4_compatible_configurations.append(config) - if args.distro == "azl4": - configurations = azl4_compatible_configurations - else: - configurations = non_azl4_compatible_configurations - if args.skipEncryptionTests: log.info(f"Skipping encryption tests as per the argument --skipEncryptionTests") non_encryption_configurations = [] diff --git a/tests/e2e_tests/target-configurations-azl4.yaml b/tests/e2e_tests/target-configurations-azl4.yaml new file mode 100644 index 0000000000..f3685f6cd0 --- /dev/null +++ b/tests/e2e_tests/target-configurations-azl4.yaml @@ -0,0 +1,54 @@ +# bareMetal: +# host: +# daily: +# - base-azl4 +# - rollback-azl4 +# validation: +# - base-azl4 +# - rollback-azl4 +# weekly: +# - base-azl4 +# - rollback-azl4 +# container: +# daily: +# - base-azl4 +# - rollback-azl4 +# validation: +# - base-azl4 +# - rollback-azl4 +# weekly: +# - base-azl4 +# - rollback-azl4 +virtualMachine: + host: + daily: + - base-azl4 + - rollback-azl4 + post_merge: + - base-azl4 + - rollback-azl4 + pullrequest: + - base-azl4 + - rollback-azl4 + validation: + - base-azl4 + - rollback-azl4 + weekly: + - base-azl4 + - rollback-azl4 + # container: + # daily: + # - base-azl4 + # - rollback-azl4 + # post_merge: + # - base-azl4 + # - rollback-azl4 + # pullrequest: + # - base-azl4 + # - rollback-azl4 + # validation: + # - base-azl4 + # - rollback-azl4 + # weekly: + # - base-azl4 + # - rollback-azl4 diff --git a/tests/e2e_tests/target-configurations.yaml b/tests/e2e_tests/target-configurations.yaml index a737487f5a..017c0eb7f2 100644 --- a/tests/e2e_tests/target-configurations.yaml +++ b/tests/e2e_tests/target-configurations.yaml @@ -87,7 +87,6 @@ virtualMachine: host: daily: - base - - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -107,7 +106,6 @@ virtualMachine: - usr-verity-raid post_merge: - base - - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -127,7 +125,6 @@ virtualMachine: - usr-verity-raid pullrequest: - base - - base-azl4 - combined - extensions - health-checks-install @@ -136,14 +133,12 @@ virtualMachine: - raid-resync-small - rerun - root-verity - - rollback-azl4 - simple - split - usr-verity - usr-verity-raid validation: - base - - base-azl4 - combined - encrypted-partition - encrypted-raid @@ -163,7 +158,6 @@ virtualMachine: - usr-verity-raid weekly: - base - - base-azl4 - combined - encrypted-partition - encrypted-raid From ece7850bf315a61aa668a4410a4d7cce26519651 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 21:41:38 +0000 Subject: [PATCH 06/19] try --- tests/e2e_tests/helpers/edit_host_config.py | 9 ++++++--- tests/images/trident-testimage/base/baseimg-azl4.yaml | 11 +++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/e2e_tests/helpers/edit_host_config.py b/tests/e2e_tests/helpers/edit_host_config.py index 0f425dc1cc..eb1e22fdc6 100644 --- a/tests/e2e_tests/helpers/edit_host_config.py +++ b/tests/e2e_tests/helpers/edit_host_config.py @@ -28,9 +28,12 @@ def add_key(host_config_path, public_key): with open(host_config_path, "r") as f: host_config = yaml.safe_load(f) - for index_user in range(len(host_config["os"]["users"])): - if host_config["os"]["users"][index_user]["name"] == "testing-user": - host_config["os"]["users"][index_user]["sshPublicKeys"].append(public_key) + if "os" in host_config and "users" in host_config.get("os", {}): + for index_user in range(len(host_config["os"]["users"])): + if host_config["os"]["users"][index_user]["name"] == "testing-user": + host_config["os"]["users"][index_user]["sshPublicKeys"].append( + public_key + ) with open(host_config_path, "w") as f: yaml.safe_dump(host_config, f) diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index fed5a7e3dd..3e20c2415b 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -27,15 +27,20 @@ storage: os: bootloader: resetType: hard-reset - hostname: trident-testimg + hostname: trident-azll4-testimg + + selinux: + mode: disabled kernelCommandLine: - # Replicates BM base image settings, that would otherwise be lost extraCommandLine: - console=tty0 - console=ttyS0 - rd.info - log_buf_len=1M + - net.ifnames=0 + - loglevel=6 + - systemd.journald.forward_to_console=1 packages: install: @@ -88,6 +93,8 @@ os: services: enable: - sshd + - systemd-networkd + - systemd-resolved - trident - tridentd.socket From a9182355ded9ca246172a21f64f3e2a5d07976d7 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 25 Jun 2026 23:33:12 +0000 Subject: [PATCH 07/19] try --- .../trident-testimage/base/baseimg-azl4.yaml | 7 +++ .../base/scripts/rebuild-initrd-azl4.sh | 62 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index 3e20c2415b..181d786431 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -113,6 +113,13 @@ scripts: postCustomization: - path: post-install.sh - path: scripts/ssh-move-host-keys-azl4.sh + # Rebuild initramfs with --no-hostonly + extra SATA drivers so the + # qcow2 boots regardless of which bus the consumer's libvirt config + # picks (storm-trident uses bus=sata; the original boot test on + # karhu-ubuntu used bus=virtio). MUST run BEFORE strip-selinux-xattrs + # because dracut writes new files with the build-time SELinux + # context, and we want those stripped too. + - path: scripts/rebuild-initrd-azl4.sh # Strip SELinux labels from all files. See the script header for the # full rationale, but in short: the AZL4 base VHDX bakes in AZL4 # SELinux contexts that Trident-from-AZL3-MOS can't preserve during diff --git a/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh b/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh new file mode 100644 index 0000000000..b07b3a8c0a --- /dev/null +++ b/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Regenerate initrd with --no-hostonly so all storage drivers are +# included, not just the ones MIC's build environment happens to need. +# +# Why: storm-trident's rollback test (tools/storm/utils/vm/qemu/qemu.go) +# attaches the qcow2 to a virt-install VM with `bus=sata`. MIC builds +# the qcow2 in a virtio-backed environment, so dracut's default +# hostonly mode produces an initramfs with only virtio drivers. On a +# SATA-backed boot, the initramfs can't find the root partition by +# UUID and systemd hangs forever waiting for /dev/disk/by-uuid/. +# +# Rebuilding with --no-hostonly bakes in ahci, ata_piix, sata_sil, etc. +# along with virtio so the same qcow2 boots regardless of the bus type +# the consumer chooses. +# +# Runs inside the MIC chroot where /sys and /proc are bind-mounted but +# the host's SELinux is not loaded (MIC strips that), so dracut's +# cp -a doesn't hit the security.selinux setxattr issue that bites in +# AZL3 MOS during install (see strip-selinux-xattrs.sh for the parallel +# write-up). + +set -euo pipefail + +# Find the kernel version installed in this image. We require exactly +# one — `ls | head -1` would silently pick the wrong one if any future +# AZL4 variant ships multiple (kernel + kernel-hyperv, extramodules-*, +# etc.). Fail loudly rather than generate an initramfs for the wrong +# kernel: the failure mode of that misstep is "boot hangs waiting for +# /dev/disk/by-uuid/", which is the exact bug this script is +# meant to prevent. +KVERS=( /usr/lib/modules/* ) +case ${#KVERS[@]} in + 0) + echo "ERROR: no kernel modules dir under /usr/lib/modules" >&2 + exit 1 + ;; + 1) + KVER=$(basename "${KVERS[0]}") + ;; + *) + echo "ERROR: expected exactly one kernel under /usr/lib/modules, found:" >&2 + printf ' %s\n' "${KVERS[@]}" >&2 + exit 1 + ;; +esac +echo "Regenerating initramfs for kernel $KVER with --no-hostonly" + +# `--no-hostonly` includes all storage modules; `--no-hostonly-cmdline` +# prevents dracut from baking the build-host's /proc/cmdline parameters +# into the initramfs (which would fight the qcow2's grub cmdline at +# runtime); `--reproducible` keeps the output bit-stable across builds +# so we can detect spurious regenerations. +dracut \ + --no-hostonly \ + --no-hostonly-cmdline \ + --reproducible \ + --add-drivers "ahci ata_piix sata_sil sata_nv sata_via sd_mod" \ + --force \ + --kver "$KVER" + +echo "Regenerated initramfs:" +ls -lh /boot/initramfs-* From b95de9b7173ad77c3c9bbc7e5537924d84c14801 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 01:57:39 +0000 Subject: [PATCH 08/19] try --- .../base-azl4/trident-config.yaml | 14 +++++++------- .../trident-testimage/base/baseimg-azl4.yaml | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml index c918e15151..8848171fe3 100644 --- a/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml +++ b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml @@ -61,13 +61,13 @@ scripts: os: selinux: mode: disabled - netplan: - version: 2 - ethernets: - vmeths: - match: - name: eth* - dhcp4: true + # netplan: + # version: 2 + # ethernets: + # vmeths: + # match: + # name: eth* + # dhcp4: true users: - name: testing-user sshPublicKeys: [] diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index 181d786431..0fb86e362f 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -61,8 +61,9 @@ os: # ^^^ - lsof - mdadm - - netplan - - netplan-default-backend-networkd + # netplan and AZL4 systemd are incompatible at boot + # - netplan + # - netplan-default-backend-networkd - openssh-server - tpm2-tools # AZL4: shim added (AZL4 ships a real signed shim chain) From d90efab187e8e7dcb30248d0536ebd6bb4e0edc4 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 17:02:05 +0000 Subject: [PATCH 09/19] use netplan again --- .../base-azl4/trident-config.yaml | 14 +++++++------- .../trident-testimage/base/baseimg-azl4.yaml | 7 ++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml index 8848171fe3..c918e15151 100644 --- a/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml +++ b/tests/e2e_tests/trident_configurations/base-azl4/trident-config.yaml @@ -61,13 +61,13 @@ scripts: os: selinux: mode: disabled - # netplan: - # version: 2 - # ethernets: - # vmeths: - # match: - # name: eth* - # dhcp4: true + netplan: + version: 2 + ethernets: + vmeths: + match: + name: eth* + dhcp4: true users: - name: testing-user sshPublicKeys: [] diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index 0fb86e362f..815b716531 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -61,9 +61,10 @@ os: # ^^^ - lsof - mdadm - # netplan and AZL4 systemd are incompatible at boot - # - netplan - # - netplan-default-backend-networkd + # AZL4 netplan and AZL4 systemd are incompatible at boot ... use + # cached netplan RPMs built from main + - netplan + - netplan-default-backend-networkd - openssh-server - tpm2-tools # AZL4: shim added (AZL4 ships a real signed shim chain) From 65b7396cbfd3cad7248bf4ba71a7a5c2b8a743bc Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 17:58:10 +0000 Subject: [PATCH 10/19] enable new netplan service; enable rollback netplan testing --- tests/images/trident-testimage/base/baseimg-azl4.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index 815b716531..ef7c46da51 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -99,6 +99,9 @@ os: - systemd-resolved - trident - tridentd.socket + # netplan-main (generate/configure split) defers virtual-device + # creation to this service; Fedora ships it preset-disabled. + - netplan-configure.service additionalFiles: # AZL4 lacks a /usr/bin/hostname binary; the pytest framework smoke- From bd9b8d3847ca55439ba68bfd794a3c725527c214 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 19:33:51 +0000 Subject: [PATCH 11/19] test(azl4): generate sshd host keys on /var/srv for host-test image The clean-install host-test image (baseimg-azl4.yaml) relocates the sshd HostKey to /var/srv/etc/ssh via ssh-move-host-keys-azl4.sh, but the srv partition is created fresh on every clean install, so no host keys exist there at first boot. sshd then exits with "no hostkeys available -- exiting" and nothing listens on TCP :22, so the "Check for trace file" step fails with "connect: connection refused" even though deployment (verified over the serial console) succeeds. Add regen-sshd-keys.service and its enable script, mirroring the grubazl4 rollback test image, to generate the host keys under /var/srv on first boot before sshd starts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../trident-testimage/base/baseimg-azl4.yaml | 10 ++++++++++ .../base/files/regen-sshd-keys.service | 14 ++++++++++++++ .../base/scripts/enable-regen-sshd-keys.sh | 7 +++++++ 3 files changed, 31 insertions(+) create mode 100644 tests/images/trident-testimage/base/files/regen-sshd-keys.service create mode 100755 tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index ef7c46da51..b9f2859835 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -113,11 +113,21 @@ os: # grpc-client commit. - source: files/use-grpc-client-commit.conf destination: /etc/systemd/system/trident.service.d/override.conf + # ssh-move-host-keys-azl4.sh points sshd HostKey at /var/srv/etc/ssh. + # The srv partition is created fresh on every clean install, so the + # host keys must be generated there on first boot (see + # regen-sshd-keys.service), otherwise sshd exits with no hostkeys + # available and nothing listens on TCP :22. + - source: files/regen-sshd-keys.service + destination: /etc/systemd/system/regen-sshd-keys.service scripts: postCustomization: - path: post-install.sh - path: scripts/ssh-move-host-keys-azl4.sh + # Enable regen-sshd-keys.service via a wants symlink so sshd has + # host keys in /var/srv on first boot. + - path: scripts/enable-regen-sshd-keys.sh # Rebuild initramfs with --no-hostonly + extra SATA drivers so the # qcow2 boots regardless of which bus the consumer's libvirt config # picks (storm-trident uses bus=sata; the original boot test on diff --git a/tests/images/trident-testimage/base/files/regen-sshd-keys.service b/tests/images/trident-testimage/base/files/regen-sshd-keys.service new file mode 100644 index 0000000000..0fe938ddc3 --- /dev/null +++ b/tests/images/trident-testimage/base/files/regen-sshd-keys.service @@ -0,0 +1,14 @@ +[Unit] +Description=Generate sshd host keys in /var/srv on first boot +ConditionPathExists=!/var/srv/etc/ssh/ssh_host_ed25519_key +Before=sshd.service +After=local-fs.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=/usr/bin/mkdir -p /var/srv/etc/ssh +ExecStart=/usr/bin/ssh-keygen -A -f /var/srv -q + +[Install] +WantedBy=multi-user.target diff --git a/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh b/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh new file mode 100755 index 0000000000..bdf901cd2e --- /dev/null +++ b/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# regen-sshd-keys is a one-shot service that generates SSH host keys in +# /var/srv on first boot. Enable it via wants symlink because the generic +# `services.enable` in MIC config is reserved for systemd unit names that +# come from packages, and our unit is delivered via additionalFiles. +ln -sf /etc/systemd/system/regen-sshd-keys.service \ + /etc/systemd/system/multi-user.target.wants/regen-sshd-keys.service From 2e228491747813d86e63985bf87394603f1560f4 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 20:14:28 +0000 Subject: [PATCH 12/19] base for azl4, remove rollback-azl4 --- tests/e2e_tests/azl4_test.py | 7 ++ .../e2e_tests/target-configurations-azl4.yaml | 16 ---- .../base-azl4/test-selection.yaml | 6 +- .../rollback-azl4/test-selection.yaml | 3 - .../rollback-azl4/trident-config.yaml | 80 ------------------- 5 files changed, 10 insertions(+), 102 deletions(-) create mode 100644 tests/e2e_tests/azl4_test.py delete mode 100644 tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml delete mode 100644 tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml diff --git a/tests/e2e_tests/azl4_test.py b/tests/e2e_tests/azl4_test.py new file mode 100644 index 0000000000..ef7ed36389 --- /dev/null +++ b/tests/e2e_tests/azl4_test.py @@ -0,0 +1,7 @@ +import pytest + +pytestmark = [pytest.mark.azl4] + + +def test_azl4() -> None: + print("AZL4") diff --git a/tests/e2e_tests/target-configurations-azl4.yaml b/tests/e2e_tests/target-configurations-azl4.yaml index f3685f6cd0..b1793e8df4 100644 --- a/tests/e2e_tests/target-configurations-azl4.yaml +++ b/tests/e2e_tests/target-configurations-azl4.yaml @@ -2,53 +2,37 @@ # host: # daily: # - base-azl4 -# - rollback-azl4 # validation: # - base-azl4 -# - rollback-azl4 # weekly: # - base-azl4 -# - rollback-azl4 # container: # daily: # - base-azl4 -# - rollback-azl4 # validation: # - base-azl4 -# - rollback-azl4 # weekly: # - base-azl4 -# - rollback-azl4 virtualMachine: host: daily: - base-azl4 - - rollback-azl4 post_merge: - base-azl4 - - rollback-azl4 pullrequest: - base-azl4 - - rollback-azl4 validation: - base-azl4 - - rollback-azl4 weekly: - base-azl4 - - rollback-azl4 # container: # daily: # - base-azl4 - # - rollback-azl4 # post_merge: # - base-azl4 - # - rollback-azl4 # pullrequest: # - base-azl4 - # - rollback-azl4 # validation: # - base-azl4 - # - rollback-azl4 # weekly: # - base-azl4 - # - rollback-azl4 diff --git a/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml index eb1a3aca31..b724409802 100644 --- a/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml +++ b/tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml @@ -1,5 +1,5 @@ compatible: + # identify configuration as azl4 - azl4 - # Reuse the same pytest assertions as the AZL3 `base` scenario where - # appropriate. Add this scenario explicitly to test markers as we wire - # up pytest coverage. + # Use the shared pytest assertions where appropriate. + - base diff --git a/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml b/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml deleted file mode 100644 index 2ea207a640..0000000000 --- a/tests/e2e_tests/trident_configurations/rollback-azl4/test-selection.yaml +++ /dev/null @@ -1,3 +0,0 @@ -compatible: - - azl4 - - rollback diff --git a/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml b/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml deleted file mode 100644 index 67a2c574a6..0000000000 --- a/tests/e2e_tests/trident_configurations/rollback-azl4/trident-config.yaml +++ /dev/null @@ -1,80 +0,0 @@ -image: - url: http://NETLAUNCH_HOST_ADDRESS/files/regular.cosi - sha384: ignored -storage: - disks: - - id: os - device: /dev/disk/by-path/pci-0000:00:1f.2-ata-2 - partitionTableType: gpt - partitions: - - id: root-a - type: root - size: 8G - - id: root-b - type: root - size: 8G - - id: esp - type: esp - size: 1G - - id: trident - type: linux-generic - size: 1G - - id: disk2 - device: /dev/disk/by-path/pci-0000:00:1f.2-ata-3 - partitionTableType: gpt - partitions: [] - abUpdate: - volumePairs: - - id: root - volumeAId: root-a - volumeBId: root-b - filesystems: - - deviceId: trident - source: new - mountPoint: /var/lib/trident - - deviceId: esp - mountPoint: - path: /boot/efi - options: umask=0077 - - deviceId: root - mountPoint: / -os: - additionalFiles: - - destination: /var/lib/trident/local-health-check-file.sh - content: | - echo 'This is a local health check script.' - exit 0 -health: - checks: - - name: invoke-rollback-from-local-script - runOn: - - clean-install - path: /var/lib/trident/local-health-check-file.sh - - name: invoke-rollback-from-script - runOn: - - clean-install - content: | - exit 1 - - name: install-failure-systemd-check - runOn: - - clean-install - systemdServices: - - non-existent-service1.service - - non-existent-service2.service - timeoutSeconds: 15 -# AZL4 variant of the AZL3 `health-checks-install/` scenario. -# - No `users`/`selinux`/`netplan` — these are baked into the test image -# at MIC build time. -# - `os.additionalFiles` is used because health.checks references -# `path: /var/lib/trident/local-health-check-file.sh`, which needs to -# be on the target filesystem. -# -# Health-check failure expectations (asserted by tests/e2e_tests/rollback_test.py): -# - State transitions to `not-provisioned` (clean-install has no slot to -# roll back to; the install just fails). -# - `/var/lib/trident/trident-health-check-failure-*.log` is created. -# - The log contains: -# * `"Failed health check(s)"` -# * `"Script 'invoke-rollback-from-script' failed"` -# * `"Unit non-existent-service1.service could not be found"` -# * `"Unit non-existent-service2.service could not be found"` From 097525ce882cc14f45fe704401a3896fef957223 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 20:18:48 +0000 Subject: [PATCH 13/19] try --- .../templates/stages/testing_rollback/testing-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/testing_rollback/testing-template.yml b/.pipelines/templates/stages/testing_rollback/testing-template.yml index f197d36253..a989ccd58d 100644 --- a/.pipelines/templates/stages/testing_rollback/testing-template.yml +++ b/.pipelines/templates/stages/testing_rollback/testing-template.yml @@ -199,7 +199,7 @@ jobs: --force-cleanup displayName: "Rollback test (${{ parameters.platform }})" workingDirectory: $(TRIDENT_SOURCE_DIR) - timeoutInMinutes: 10 + timeoutInMinutes: 40 - bash: | set -eux From 47a98ce6db74d75034d41dff3e872f0b5047b5d5 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 21:11:22 +0000 Subject: [PATCH 14/19] storm/sftp: auto-detect sftp-server path for AZL4 SudoSFTP hard-coded the AZL3 sftp-server path (/usr/libexec/sftp-server). On AZL4 (Fedora-based openssh) the binary lives at /usr/libexec/openssh/sftp-server, so `sudo -n /usr/libexec/sftp-server` failed (binary not found), the exec channel closed, and the SFTP client errored at the protocol handshake: failed to create SFTP client: error receiving version packet from server: server unexpectedly closed connection: unexpected EOF This broke the host-test ab-update helper update-hc step (Stage and finalize A/B update into target OS B) on azl4, while plain SSH exec (get-config) still worked. Exec the first existing sftp-server path (openssh/, plain libexec, or the Debian/Ubuntu location) so the SFTP protocol flows regardless of distro. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/utils/ssh/sftp/sftp.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/storm/utils/ssh/sftp/sftp.go b/tools/storm/utils/ssh/sftp/sftp.go index c59d291a8d..6ffec58611 100644 --- a/tools/storm/utils/ssh/sftp/sftp.go +++ b/tools/storm/utils/ssh/sftp/sftp.go @@ -9,8 +9,16 @@ import ( ) const ( - AZL3_SFTP_SERVER_PATH = "/usr/libexec/sftp-server" - AZL3_SFTP_SERVER_CMD = "sudo -n " + AZL3_SFTP_SERVER_PATH + // sftp-server is installed under different libexec paths depending on the + // distro / openssh packaging: + // - AZL3 / upstream openssh: /usr/libexec/sftp-server + // - AZL4 (Fedora-based) / RHEL: /usr/libexec/openssh/sftp-server + // - Debian / Ubuntu: /usr/lib/openssh/sftp-server + // Exec the first path that exists so the SFTP protocol speaks over + // stdin/stdout regardless of where the binary lives. Hard-coding the AZL3 + // path made SudoSFTP fail on AZL4 (binary not found -> channel closes -> + // "error receiving version packet from server: unexpected EOF"). + SFTP_SERVER_CMD = `sudo -n sh -c 'for p in /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server /usr/lib/openssh/sftp-server; do [ -x "$p" ] && exec "$p"; done; echo "sftp-server not found" >&2; exit 127'` ) type SftpSudoClient struct { @@ -46,9 +54,9 @@ func NewSftpSudoClient(client *ssh.Client, opts ...sftp.ClientOption) (*SftpSudo return nil, fmt.Errorf("failed to create SSH session: %w", err) } - ok, err := session.SendRequest("exec", true, ssh.Marshal(struct{ Command string }{AZL3_SFTP_SERVER_CMD})) + ok, err := session.SendRequest("exec", true, ssh.Marshal(struct{ Command string }{SFTP_SERVER_CMD})) if err == nil && !ok { - err = fmt.Errorf("sftp: command %v failed", AZL3_SFTP_SERVER_CMD) + err = fmt.Errorf("sftp: command %v failed", SFTP_SERVER_CMD) } if err != nil { return nil, err From f4efac576bc8fc594c69223efc587b4bd875d02b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 26 Jun 2026 21:15:09 +0000 Subject: [PATCH 15/19] optional dummy interface removed commit delays --- .../templates/stages/testing_rollback/testing-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/testing_rollback/testing-template.yml b/.pipelines/templates/stages/testing_rollback/testing-template.yml index a989ccd58d..f197d36253 100644 --- a/.pipelines/templates/stages/testing_rollback/testing-template.yml +++ b/.pipelines/templates/stages/testing_rollback/testing-template.yml @@ -199,7 +199,7 @@ jobs: --force-cleanup displayName: "Rollback test (${{ parameters.platform }})" workingDirectory: $(TRIDENT_SOURCE_DIR) - timeoutInMinutes: 40 + timeoutInMinutes: 10 - bash: | set -eux From 1d1292021b2f91cd02658007741192d5b5b138cc Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 27 Jun 2026 02:14:15 +0000 Subject: [PATCH 16/19] azl4: address PR review feedback on testimage scripts - strip-selinux-xattrs.sh: test setfattr exit status directly and match benign errors with a shell case (no per-entry grep, no stale rc). - enable-regen-sshd-keys.sh: set -euo pipefail and mkdir -p the wants dir so enabling fails fast and works in minimal chroots. - regen-sshd-keys.service: OR-style ConditionPathExists for rsa/ecdsa/ ed25519 so any missing host key triggers regeneration. - rebuild-initrd-azl4.sh: shopt -s nullglob so a missing/empty modules dir yields an empty array and hits the 0) error branch. - baseimg-azl4.yaml: fix hostname typo azll4 -> azl4. - edit_host_config.py: fail loudly when os.users or testing-user is missing instead of silently skipping the SSH key. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/e2e_tests/helpers/edit_host_config.py | 24 ++++++++++++++----- .../trident-testimage/base/baseimg-azl4.yaml | 2 +- .../base/files/regen-sshd-keys.service | 4 +++- .../base/scripts/enable-regen-sshd-keys.sh | 2 ++ .../base/scripts/rebuild-initrd-azl4.sh | 3 +++ .../base/scripts/strip-selinux-xattrs.sh | 20 ++++++++++------ 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/e2e_tests/helpers/edit_host_config.py b/tests/e2e_tests/helpers/edit_host_config.py index eb1e22fdc6..67e0766329 100644 --- a/tests/e2e_tests/helpers/edit_host_config.py +++ b/tests/e2e_tests/helpers/edit_host_config.py @@ -28,12 +28,24 @@ def add_key(host_config_path, public_key): with open(host_config_path, "r") as f: host_config = yaml.safe_load(f) - if "os" in host_config and "users" in host_config.get("os", {}): - for index_user in range(len(host_config["os"]["users"])): - if host_config["os"]["users"][index_user]["name"] == "testing-user": - host_config["os"]["users"][index_user]["sshPublicKeys"].append( - public_key - ) + users = host_config.get("os", {}).get("users") + if not users: + raise ValueError( + f"{host_config_path}: expected os.users to be present so the test " + "SSH key can be added, but it is missing or empty" + ) + + added = False + for user in users: + if user.get("name") == "testing-user": + user.setdefault("sshPublicKeys", []).append(public_key) + added = True + + if not added: + raise ValueError( + f"{host_config_path}: no os.users entry named 'testing-user' was " + "found to add the test SSH key to" + ) with open(host_config_path, "w") as f: yaml.safe_dump(host_config, f) diff --git a/tests/images/trident-testimage/base/baseimg-azl4.yaml b/tests/images/trident-testimage/base/baseimg-azl4.yaml index b9f2859835..66236f3ea7 100644 --- a/tests/images/trident-testimage/base/baseimg-azl4.yaml +++ b/tests/images/trident-testimage/base/baseimg-azl4.yaml @@ -27,7 +27,7 @@ storage: os: bootloader: resetType: hard-reset - hostname: trident-azll4-testimg + hostname: trident-azl4-testimg selinux: mode: disabled diff --git a/tests/images/trident-testimage/base/files/regen-sshd-keys.service b/tests/images/trident-testimage/base/files/regen-sshd-keys.service index 0fe938ddc3..e33478caae 100644 --- a/tests/images/trident-testimage/base/files/regen-sshd-keys.service +++ b/tests/images/trident-testimage/base/files/regen-sshd-keys.service @@ -1,6 +1,8 @@ [Unit] Description=Generate sshd host keys in /var/srv on first boot -ConditionPathExists=!/var/srv/etc/ssh/ssh_host_ed25519_key +ConditionPathExists=|!/var/srv/etc/ssh/ssh_host_rsa_key +ConditionPathExists=|!/var/srv/etc/ssh/ssh_host_ecdsa_key +ConditionPathExists=|!/var/srv/etc/ssh/ssh_host_ed25519_key Before=sshd.service After=local-fs.target diff --git a/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh b/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh index bdf901cd2e..feb05323f3 100755 --- a/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh +++ b/tests/images/trident-testimage/base/scripts/enable-regen-sshd-keys.sh @@ -3,5 +3,7 @@ # /var/srv on first boot. Enable it via wants symlink because the generic # `services.enable` in MIC config is reserved for systemd unit names that # come from packages, and our unit is delivered via additionalFiles. +set -euo pipefail +mkdir -p /etc/systemd/system/multi-user.target.wants ln -sf /etc/systemd/system/regen-sshd-keys.service \ /etc/systemd/system/multi-user.target.wants/regen-sshd-keys.service diff --git a/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh b/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh index b07b3a8c0a..38212b4c0e 100644 --- a/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh +++ b/tests/images/trident-testimage/base/scripts/rebuild-initrd-azl4.sh @@ -28,6 +28,9 @@ set -euo pipefail # kernel: the failure mode of that misstep is "boot hangs waiting for # /dev/disk/by-uuid/", which is the exact bug this script is # meant to prevent. +# nullglob so a missing/empty /usr/lib/modules yields an empty array +# (otherwise the glob stays literal and the 0) branch never matches). +shopt -s nullglob KVERS=( /usr/lib/modules/* ) case ${#KVERS[@]} in 0) diff --git a/tests/images/trident-testimage/base/scripts/strip-selinux-xattrs.sh b/tests/images/trident-testimage/base/scripts/strip-selinux-xattrs.sh index aaa8f38440..8fa1c159fb 100644 --- a/tests/images/trident-testimage/base/scripts/strip-selinux-xattrs.sh +++ b/tests/images/trident-testimage/base/scripts/strip-selinux-xattrs.sh @@ -47,16 +47,22 @@ fail_count=0 while IFS= read -r -d '' f; do # Capture stderr so we can distinguish ENODATA ("no such attribute", # benign — nothing to strip) from real failures (EPERM, EOPNOTSUPP). - err=$(setfattr -h -x security.selinux "$f" 2>&1 >/dev/null) || rc=$? && rc=${rc:-0} - if [ "$rc" -eq 0 ]; then + # Test setfattr's exit status directly in the if-condition so rc never + # carries a stale value between iterations, and match the benign-error + # strings with a shell `case` so we don't fork a grep per entry. + if err=$(setfattr -h -x security.selinux "$f" 2>&1 >/dev/null); then count=$((count + 1)) - elif echo "$err" | grep -qE "No such attribute|Operation not supported"; then - : # nothing to strip, expected for files without the xattr else - fail_count=$((fail_count + 1)) - echo "setfattr failed on '$f': $err" >&2 + case "$err" in + *"No such attribute"*|*"Operation not supported"*) + : # nothing to strip, expected for files without the xattr + ;; + *) + fail_count=$((fail_count + 1)) + echo "setfattr failed on '$f': $err" >&2 + ;; + esac fi - rc=0 done < <(find / \( -path /proc -o -path /sys -o -path /dev -o -path /run \) -prune \ -o \( -type f -o -type d -o -type l \) -print0) From 31755f52e86a6cc921d0979ebad6afb1b43d2ba1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 27 Jun 2026 02:22:43 +0000 Subject: [PATCH 17/19] azl4: register azl4 pytest marker Register the azl4 marker in tests/e2e_tests/pytest.ini so pytestmark in azl4_test.py no longer emits PytestUnknownMarkWarning and stays compatible with a future --strict-markers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/e2e_tests/pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e_tests/pytest.ini b/tests/e2e_tests/pytest.ini index 1301865bfa..0435c13fc8 100644 --- a/tests/e2e_tests/pytest.ini +++ b/tests/e2e_tests/pytest.ini @@ -7,6 +7,7 @@ markers = encryption: Tests designed to verify the encryption feature ops in Trident. ab_update_staged: Tests designed to verify that A/B update was staged correctly. extensions: Tests designed to verify that the servicing of sysexts and confexts succeeded. + azl4: Tests designed to verify Azure Linux 4.0 (AZL4) base image support. # Special markers, do not use for tests, specify by Trident configuration: compatible: Tests that are compatible with a Trident configuration. From 09699d81079e84b5a6f3cf21e785c186b1985139 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 27 Jun 2026 02:37:45 +0000 Subject: [PATCH 18/19] storm/sftp: invoke sftp-server shell via absolute path Use `sudo -n -- /bin/sh -c ...` instead of `sudo -n sh -c ...` so the SudoSFTP handshake does not depend on `sh` being discoverable via sudo's secure_path, which can be restricted on some sudoers configs and reintroduce the "unexpected EOF" handshake failure. `--` also ends sudo option parsing defensively. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/utils/ssh/sftp/sftp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/storm/utils/ssh/sftp/sftp.go b/tools/storm/utils/ssh/sftp/sftp.go index 6ffec58611..b6cbbcfa2d 100644 --- a/tools/storm/utils/ssh/sftp/sftp.go +++ b/tools/storm/utils/ssh/sftp/sftp.go @@ -18,7 +18,7 @@ const ( // stdin/stdout regardless of where the binary lives. Hard-coding the AZL3 // path made SudoSFTP fail on AZL4 (binary not found -> channel closes -> // "error receiving version packet from server: unexpected EOF"). - SFTP_SERVER_CMD = `sudo -n sh -c 'for p in /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server /usr/lib/openssh/sftp-server; do [ -x "$p" ] && exec "$p"; done; echo "sftp-server not found" >&2; exit 127'` + SFTP_SERVER_CMD = `sudo -n -- /bin/sh -c 'for p in /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server /usr/lib/openssh/sftp-server; do [ -x "$p" ] && exec "$p"; done; echo "sftp-server not found" >&2; exit 127'` ) type SftpSudoClient struct { From 3175772c629080069e14bc6fef186b3e94e311ff Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 27 Jun 2026 02:47:41 +0000 Subject: [PATCH 19/19] azl4: harden user test skip guard and sftp sudo scope - base_test.py: skip test_users when os.users is missing OR empty/null, not just when the key is absent. Avoids silently passing with no assertions on an empty list and avoids iterating over None. - storm/sftp: run the path-probing shell unprivileged and only exec the chosen sftp-server binary under sudo (`exec sudo -n -- "$p"`), so the handshake no longer requires a root-shell sudoers entry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/e2e_tests/base_test.py | 5 +++-- tools/storm/utils/ssh/sftp/sftp.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/e2e_tests/base_test.py b/tests/e2e_tests/base_test.py index 884e2dcec1..95554175d3 100644 --- a/tests/e2e_tests/base_test.py +++ b/tests/e2e_tests/base_test.py @@ -421,10 +421,11 @@ def test_users(connection, hostConfiguration): expected_users = list() expected_groups = dict() - if "os" not in hostConfiguration or "users" not in hostConfiguration.get("os", {}): + users = hostConfiguration.get("os", {}).get("users") + if not users: pytest.skip("No os.users in trident config (user baked into image by MIC)") - for user_info in hostConfiguration["os"]["users"]: + for user_info in users: expected_users.append(user_info["name"]) if "groups" in user_info: for group in user_info["groups"]: diff --git a/tools/storm/utils/ssh/sftp/sftp.go b/tools/storm/utils/ssh/sftp/sftp.go index b6cbbcfa2d..98514c21d7 100644 --- a/tools/storm/utils/ssh/sftp/sftp.go +++ b/tools/storm/utils/ssh/sftp/sftp.go @@ -18,7 +18,7 @@ const ( // stdin/stdout regardless of where the binary lives. Hard-coding the AZL3 // path made SudoSFTP fail on AZL4 (binary not found -> channel closes -> // "error receiving version packet from server: unexpected EOF"). - SFTP_SERVER_CMD = `sudo -n -- /bin/sh -c 'for p in /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server /usr/lib/openssh/sftp-server; do [ -x "$p" ] && exec "$p"; done; echo "sftp-server not found" >&2; exit 127'` + SFTP_SERVER_CMD = `/bin/sh -c 'for p in /usr/libexec/openssh/sftp-server /usr/libexec/sftp-server /usr/lib/openssh/sftp-server; do [ -x "$p" ] && exec sudo -n -- "$p"; done; echo "sftp-server not found" >&2; exit 127'` ) type SftpSudoClient struct {