azl4: enable e2e and rollback pipeline testing

.pipelines/templates/e2e-template.yml

@@ -181,6 +181,15 @@ stages:
       micVersion: ${{ parameters.micVersion }}
       dependsOnStage: ${{ parameters.baseImageArtifactStage }}
 
+  # Build Trident test image (regular) for AZL4
+  - template: stages/build_image/build-image.yml
+    parameters:
+      imageName: trident-azl4-testimage
+      micBuildType: ${{ parameters.micBuildType }}
+      micVersion: ${{ parameters.micVersion }}
+      dependsOnStage: ${{ parameters.baseImageArtifactStage }}
+      azureLinuxVersion: "4.0-preview"
+
   # Build Trident test image (container)
   - template: stages/build_image/build-image.yml
     parameters:
@@ -409,6 +418,15 @@ stages:
           buildPurpose: "pullrequest"
           runtimeEnv: "container"
 
+      # VM Testing (host, pullrequest) for AZL4
+      - template: stages/testing_vm/netlaunch-testing.yml
+        parameters:
+          buildPurpose: "pullrequest"
+          runtimeEnv: "host"
+          distro: azl4
+          stageSuffix: "_azl4"
+          testSecureBoot: false
+
       # Run direct streaming tests for amd64 with streaming images
       - template: direct-streaming-test.yml
         parameters:

.pipelines/templates/stages/common_tasks/push-to-acr.yml

@@ -17,6 +17,7 @@ parameters:
     type: string
     values:
       - trident-testimage
+      - trident-azl4-testimage
       - trident-container-testimage
 
   - name: "config"

.pipelines/templates/stages/common_tasks/remove-from-acr.yml

@@ -17,6 +17,7 @@ parameters:
     type: string
     values:
       - trident-testimage
+      - trident-azl4-testimage
       - trident-container-testimage
 
   - name: "config"

.pipelines/templates/stages/testing_common/download-test-images.yml

@@ -19,6 +19,7 @@ parameters:
     values:
       - trident-container-testimage
       - trident-testimage
+      - trident-azl4-testimage
       - azurelinux-direct-streaming-testimage-amd64
       - azurelinux-direct-streaming-testimage-arm64
       - ubuntu-direct-streaming-testimage-2204-amd64

.pipelines/templates/stages/testing_common/get-tests.yml

@@ -31,6 +31,13 @@ parameters:
     type: boolean
     default: true
 
+  - name: distro
+    type: string
+    default: "azl3"
+    values:
+      - azl3
+      - azl4
+
 
 jobs:
   - job: DefineTests
@@ -51,8 +58,12 @@ jobs:
           if [ "${{ parameters.testSecureBoot }}" == "False" ]; then
             SKIP_ENCRYPTION_TESTS_ARG="--skipEncryptionTests"
           fi
+          CONFIGURATIONS_FILE=./tests/e2e_tests/target-configurations.yaml
+          if [ "${{ parameters.distro }}" == "azl4" ]; then
+            CONFIGURATIONS_FILE=./tests/e2e_tests/target-configurations-azl4.yaml
+          fi
           python3 ./tests/e2e_tests/helpers/read_target_configurations.py \
-              --configurations ./tests/e2e_tests/target-configurations.yaml \
+              --configurations $CONFIGURATIONS_FILE \
               --env ${{ parameters.deploymentEnvironment }} \
               --runtimeEnv ${{ parameters.runtimeEnv }} \
               --purpose ${{ parameters.buildPurpose }} \

.pipelines/templates/stages/testing_common/trident-prep.yml

@@ -19,6 +19,7 @@ parameters:
     default: trident-testimage
     values:
       - trident-testimage
+      - trident-azl4-testimage
       - trident-container-testimage
 
   - name: config

.pipelines/templates/stages/testing_vm/netlaunch-testing.yml

@@ -39,37 +39,66 @@ parameters:
       - maritimus-dev-acr-write-umi
       - trident-dev-acr-write-umi-ECF
 
+  - name: distro
+    type: string
+    default: azl3
+    values:
+      - azl3
+      - azl4
+
+  - name: stageSuffix
+    type: string
+    default: ""
+
 stages:
-  - stage: DefineTests_VM_${{ parameters.runtimeEnv }}
-    displayName: Test List for VM:${{ parameters.runtimeEnv }}
+  - stage: DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }}
+    displayName: Test List for VM:${{ parameters.runtimeEnv }} (${{ parameters.distro }})
     jobs:
       - template: ../testing_common/get-tests.yml
         parameters:
           buildPurpose: ${{ parameters.buildPurpose }}
           deploymentEnvironment: virtualMachine
           runtimeEnv: ${{ parameters.runtimeEnv }}
           testSecureBoot: ${{ parameters.testSecureBoot }}
+          distro: ${{ parameters.distro }}
 
-  - stage: DeploymentTesting_${{ parameters.runtimeEnv }}
-    displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing
+  - stage: DeploymentTesting_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }}
+    displayName: Deployment VM ${{ parameters.runtimeEnv }} Testing (${{ parameters.distro }})
     dependsOn:
-      - DefineTests_VM_${{ parameters.runtimeEnv }}
+      - DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }}
       - ${{ if eq(parameters.testingRun, true) }}:
           - DownloadTestingElements
       - ${{ else }}:
           - BuildingTools
-          - ${{ if eq(parameters.runtimeEnv, 'container') }}:
-              - BuildTridentContainerImage
-              - TridentTestImg_trident_container_installer
-              - TridentTestImg_trident_container_testimage
-              - TridentTestImg_trident_container_verity_testimage
-              - TridentTestImg_trident_container_usrverity_testimage
-          - ${{ else }}:
-              - TridentTestImg_trident_split_installer
-              - TridentTestImg_trident_installer
-              - TridentTestImg_trident_testimage
-              - TridentTestImg_trident_verity_testimage
-              - TridentTestImg_trident_usrverity_testimage
+          - ${{ if eq(parameters.distro, 'azl3') }}:
+            - ${{ if eq(parameters.runtimeEnv, 'container') }}:
+                - BuildTridentContainerImage
+                - TridentTestImg_trident_container_installer
+                - TridentTestImg_trident_container_testimage
+                - TridentTestImg_trident_container_verity_testimage
+                - TridentTestImg_trident_container_usrverity_testimage
+            - ${{ else }}:
+                - TridentTestImg_trident_split_installer
+                - TridentTestImg_trident_installer
+                - TridentTestImg_trident_testimage
+                - TridentTestImg_trident_verity_testimage
+                - TridentTestImg_trident_usrverity_testimage
+          - ${{ if eq(parameters.distro, 'azl4') }}:
+            - ${{ if eq(parameters.runtimeEnv, 'container') }}:
+                # TODO: the following are placeholders until container supported for AZL4
+                - BuildTridentContainerImage
+                - TridentTestImg_trident_container_installer
+                - TridentTestImg_trident_container_testimage
+                - TridentTestImg_trident_container_verity_testimage
+                - TridentTestImg_trident_container_usrverity_testimage
+            - ${{ else }}:
+                - TridentTestImg_trident_installer
+                - TridentTestImg_trident_azl4_testimage
+                # TODO: the following are placeholders until the verity
+                # and usrverity test images are built for AZL4
+                - TridentTestImg_trident_split_installer
+                - TridentTestImg_trident_verity_testimage
+                - TridentTestImg_trident_usrverity_testimage
 
     jobs:
       - job: Testing
@@ -80,7 +109,7 @@ stages:
           hostArchitecture: amd64
 
         strategy:
-          matrix: $[ stageDependencies.DefineTests_VM_${{ parameters.runtimeEnv }}.DefineTests.outputs['setConfigurations.matrixConfigurations'] ]
+          matrix: $[ stageDependencies.DefineTests_VM_${{ parameters.runtimeEnv }}${{ parameters.stageSuffix }}.DefineTests.outputs['setConfigurations.matrixConfigurations'] ]
 
         variables:
           # Sourced from the matrix
@@ -90,28 +119,55 @@ stages:
           - name: tridentConfigPath
             value: tests/e2e_tests/trident_configurations/$(tridentConfigurationName)
 
-          - ${{ if eq(parameters.runtimeEnv, 'container') }}:
-              - name: installerISOName
-                value: trident-container-installer
-              - name: testImageName
-                value: trident-container-testimage
-              - name: verityTestImageName
-                value: trident-container-verity-testimage
-              - name: usrVerityTestImageName
-                value: trident-container-usrverity-testimage
-              - name: downloadTridentContainer
-                value: true
-          - ${{ else }}:
-              - name: installerISOName
-                value: trident-installer
-              - name: testImageName
-                value: trident-testimage
-              - name: verityTestImageName
-                value: trident-verity-testimage
-              - name: usrVerityTestImageName
-                value: trident-usrverity-testimage
-              - name: downloadTridentContainer
-                value: false
+          - ${{ if eq(parameters.distro, 'azl3') }}:
+            - ${{ if eq(parameters.runtimeEnv, 'container') }}:
+                - name: installerISOName
+                  value: trident-container-installer
+                - name: testImageName
+                  value: trident-container-testimage
+                - name: verityTestImageName
+                  value: trident-container-verity-testimage
+                - name: usrVerityTestImageName
+                  value: trident-container-usrverity-testimage
+                - name: downloadTridentContainer
+                  value: true
+            - ${{ else }}:
+                - name: installerISOName
+                  value: trident-installer
+                - name: testImageName
+                  value: trident-testimage
+                - name: verityTestImageName
+                  value: trident-verity-testimage
+                - name: usrVerityTestImageName
+                  value: trident-usrverity-testimage
+                - name: downloadTridentContainer
+                  value: false
+          - ${{ if eq(parameters.distro, 'azl4') }}:
+            - ${{ if eq(parameters.runtimeEnv, 'container') }}:
+                # TODO: the following variables are placeholders until container supported for AZL4
+                - name: installerISOName
+                  value: trident-container-installer
+                - name: testImageName
+                  value: trident-container-testimage
+                - name: verityTestImageName
+                  value: trident-container-verity-testimage
+                - name: usrVerityTestImageName
+                  value: trident-container-usrverity-testimage
+                - name: downloadTridentContainer
+                  value: true
+            - ${{ else }}:
+                - name: installerISOName
+                  value: trident-installer
+                - name: testImageName
+                  value: trident-azl4-testimage
+                # TODO: the following variables are placeholders until the verity
+                # and usrverity test images are built for AZL4
+                - name: verityTestImageName
+                  value: trident-verity-testimage
+                - name: usrVerityTestImageName
+                  value: trident-usrverity-testimage
+                - name: downloadTridentContainer
+                  value: false
 
           - name: ob_outputDirectory
             value: /tmp/deployment_logs

tests/e2e_tests/azl4_test.py

@@ -0,0 +1,7 @@
+import pytest
+
+pytestmark = [pytest.mark.azl4]
+
+
+def test_azl4() -> None:
+    print("AZL4")

tests/e2e_tests/base_test.py

@@ -421,7 +421,11 @@ def test_users(connection, hostConfiguration):
     expected_users = list()
     expected_groups = dict()
 
-    for user_info in hostConfiguration["os"]["users"]:
+    users = hostConfiguration.get("os", {}).get("users")
+    if not users:
+        pytest.skip("No os.users in trident config (user baked into image by MIC)")
+
+    for user_info in users:
         expected_users.append(user_info["name"])
         if "groups" in user_info:
             for group in user_info["groups"]:

tests/e2e_tests/helpers/edit_host_config.py

@@ -28,9 +28,24 @@ def add_key(host_config_path, public_key):
     with open(host_config_path, "r") as f:
         host_config = yaml.safe_load(f)
 
-    for index_user in range(len(host_config["os"]["users"])):
-        if host_config["os"]["users"][index_user]["name"] == "testing-user":
-            host_config["os"]["users"][index_user]["sshPublicKeys"].append(public_key)
+    users = host_config.get("os", {}).get("users")
+    if not users:
+        raise ValueError(
+            f"{host_config_path}: expected os.users to be present so the test "
+            "SSH key can be added, but it is missing or empty"
+        )
+
+    added = False
+    for user in users:
+        if user.get("name") == "testing-user":
+            user.setdefault("sshPublicKeys", []).append(public_key)
+            added = True
+
+    if not added:
+        raise ValueError(
+            f"{host_config_path}: no os.users entry named 'testing-user' was "
+            "found to add the test SSH key to"
+        )
 
     with open(host_config_path, "w") as f:
         yaml.safe_dump(host_config, f)

tests/e2e_tests/pytest.ini

@@ -7,6 +7,7 @@ markers =
     encryption: Tests designed to verify the encryption feature ops in Trident.
     ab_update_staged: Tests designed to verify that A/B update was staged correctly.
     extensions: Tests designed to verify that the servicing of sysexts and confexts succeeded.
+    azl4: Tests designed to verify Azure Linux 4.0 (AZL4) base image support.
 
     # Special markers, do not use for tests, specify by Trident configuration:
     compatible: Tests that are compatible with a Trident configuration.

tests/e2e_tests/target-configurations-azl4.yaml

@@ -0,0 +1,38 @@
+# bareMetal:
+#   host:
+#     daily:
+#       - base-azl4
+#     validation:
+#       - base-azl4
+#     weekly:
+#       - base-azl4
+#   container:
+#     daily:
+#       - base-azl4
+#     validation:
+#       - base-azl4
+#     weekly:
+#       - base-azl4
+virtualMachine:
+  host:
+    daily:
+      - base-azl4
+    post_merge:
+      - base-azl4
+    pullrequest:
+      - base-azl4
+    validation:
+      - base-azl4
+    weekly:
+      - base-azl4
+  # container:
+  #   daily:
+  #     - base-azl4
+  #   post_merge:
+  #     - base-azl4
+  #   pullrequest:
+  #     - base-azl4
+  #   validation:
+  #     - base-azl4
+  #   weekly:
+  #     - base-azl4

tests/e2e_tests/trident_configurations/base-azl4/test-selection.yaml

@@ -0,0 +1,5 @@
+compatible:
+  # identify configuration as azl4
+  - azl4
+  # Use the shared pytest assertions where appropriate.
+  - base