Switch to CentOS Stream 10 IPA images on Rocky 10

.github/workflows/ipa-image-build.yml

@@ -11,6 +11,14 @@ on:
         description: Build Rocky Linux 9 aarch64
         type: boolean
         default: true
+      rocky10:
+        description: Build Rocky Linux 10
+        type: boolean
+        default: true
+      rocky10-aarch64:
+        description: Build Rocky Linux 10 aarch64
+        type: boolean
+        default: true
       ubuntu-noble:
         description: Build Ubuntu 24.04 Noble
         type: boolean
@@ -54,6 +62,27 @@ jobs:
       openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
       ipa_image_tag: ${{ steps.ipa_image_tag.outputs.ipa_image_tag }}
     steps:
+      - name: Validate inputs
+        run: |
+          if [[
+            "${{ inputs.rocky9 }}" == "false" &&
+            "${{ inputs.rocky9-aarch64 }}" == "false" &&
+            "${{ inputs.rocky10 }}" == "false" &&
+            "${{ inputs.rocky10-aarch64 }}" == "false" &&
+            "${{ inputs.ubuntu-noble }}" == "false"
+          ]]; then
+            echo "At least one distribution must be selected"
+            exit 1
+          fi
+
+          if [[
+            ("${{ inputs.rocky9-aarch64 }}" == "true" || "${{ inputs.rocky10-aarch64 }}" == "true") &&
+            "${{ inputs.runner_env }}" != "SMS Lab"
+          ]]; then
+            echo "aarch64 builds are only supported on SMS Lab"
+            exit 1
+          fi
+
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -72,7 +101,7 @@ jobs:
 
   ipa-image-build:
     name: Build IPA images (x86_64)
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && (inputs.rocky9 || inputs.ubuntu-noble)
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && (inputs.rocky9 || inputs.rocky10 || inputs.ubuntu-noble)
     environment: ${{ inputs.runner_env }}
     runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
     needs:
@@ -131,7 +160,7 @@ jobs:
           ssh_public_key = "id_rsa.pub"
           ssh_username = "ubuntu"
           aio_vm_name = "${{ env.VM_NAME }}"
-          aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}"
+          aio_vm_image = "Ubuntu-24.04"
           aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
           aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
           aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
@@ -229,7 +258,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed host command run \
-          --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output
+          --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
 
@@ -357,6 +386,68 @@ jobs:
           scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/rocky-9/
         if: inputs.rocky9
 
+      - name: Build a Rocky 10 IPA image
+        id: build_rocky_10_ipa
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe overcloud deployment image build --force-rebuild \
+          -e ipa_build_distro="centos" \
+          -e ipa_build_release="10-stream"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10
+
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: steps.build_rocky_10_ipa.outcome == 'failure'
+
+      - name: Upload Rocky 10 IPA kernel image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/ipa \
+          -e artifact_type=ipa-images \
+          -e artifact_tag=${{ needs.create-tag.outputs.ipa_image_tag }} \
+          -e os_distribution="rocky" \
+          -e os_release="10" \
+          -e file_regex='*.kernel'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10 && steps.build_rocky_10_ipa.outcome == 'success'
+
+      - name: Upload Rocky 10 IPA ramdisk image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/ipa \
+          -e artifact_type=ipa-images \
+          -e artifact_tag=${{ needs.create-tag.outputs.ipa_image_tag }} \
+          -e os_distribution="rocky" \
+          -e os_release="10" \
+          -e file_regex='*.initramfs'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10 && steps.build_rocky_10_ipa.outcome == 'success'
+
+      - name: Copy logs back
+        continue-on-error: true
+        run: |
+          mkdir -p logs/rocky-10
+          scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/rocky-10/
+        if: inputs.rocky10
+
       - name: Upload logs artifact
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
@@ -368,6 +459,7 @@ jobs:
           echo "Builds failed. See workflow artifacts for details." &&
           exit 1
         if: steps.build_rocky_9_ipa.outcome == 'failure' ||
+            steps.build_rocky_10_ipa.outcome == 'failure' ||
             steps.build_ubuntu_noble_ipa.outcome == 'failure'
 
       - name: Destroy
@@ -381,7 +473,7 @@ jobs:
 
   ipa-image-build-aarch64:
     name: Build IPA images (aarch64)
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.rocky9-aarch64 && inputs.runner_env == 'SMS Lab'
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && (inputs.rocky9-aarch64 || inputs.rocky10-aarch64) && inputs.runner_env == 'SMS Lab'
     environment: ${{ inputs.runner_env }}
     runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
     needs:
@@ -440,7 +532,7 @@ jobs:
           ssh_public_key = "id_rsa.pub"
           ssh_username = "ubuntu"
           aio_vm_name = "${{ env.VM_NAME }}"
-          aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE_ARM64 }}"
+          aio_vm_image = "Ubuntu-24.04-arm64"
           aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
           aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
           aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
@@ -538,7 +630,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed host command run \
-          --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output
+          --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
 
@@ -609,6 +701,73 @@ jobs:
           scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/rocky-9/
         if: inputs.rocky9-aarch64
 
+      - name: Build a Rocky 10 aarch64 IPA image
+        id: build_rocky_10_ipa_aarch64
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe overcloud deployment image build --force-rebuild \
+          -e ipa_build_distro="centos" \
+          -e ipa_build_release="10-stream" \
+          -e ipa_build_arch="aarch64"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10-aarch64
+
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: steps.build_rocky_10_ipa_aarch64.outcome == 'failure'
+
+      - name: Upload Rocky 10 aarch64 IPA kernel image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/ipa \
+          -e artifact_type=ipa-images \
+          -e artifact_tag=${{ needs.create-tag.outputs.ipa_image_tag }} \
+          -e repository_name="ipa-images-${{ needs.create-tag.outputs.openstack_release }}-rocky-10-aarch64" \
+          -e pulp_base_path="ipa-images/${{ needs.create-tag.outputs.openstack_release }}/rocky/10/aarch64" \
+          -e os_distribution="rocky" \
+          -e os_release="10" \
+          -e file_regex='ipa.kernel'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: steps.build_rocky_10_ipa_aarch64.outcome == 'success'
+
+      - name: Upload Rocky 10 aarch64 IPA ramdisk image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/ipa \
+          -e artifact_type=ipa-images \
+          -e artifact_tag=${{ needs.create-tag.outputs.ipa_image_tag }} \
+          -e repository_name="ipa-images-${{ needs.create-tag.outputs.openstack_release }}-rocky-10-aarch64" \
+          -e pulp_base_path="ipa-images/${{ needs.create-tag.outputs.openstack_release }}/rocky/10/aarch64" \
+          -e os_distribution="rocky" \
+          -e os_release="10" \
+          -e file_regex='ipa.initramfs'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: steps.build_rocky_10_ipa_aarch64.outcome == 'success'
+
+      - name: Copy logs back
+        continue-on-error: true
+        run: |
+          mkdir -p logs/rocky-10
+          scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/rocky-10/
+        if: inputs.rocky10-aarch64
+
       - name: Upload logs artifact
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
@@ -619,7 +778,7 @@ jobs:
         run: |
           echo "Build failed. See workflow artifacts for details." &&
           exit 1
-        if: steps.build_rocky_9_ipa_aarch64.outcome == 'failure'
+        if: steps.build_rocky_9_ipa_aarch64.outcome == 'failure' || steps.build_rocky_10_ipa_aarch64.outcome == 'failure'
 
       - name: Destroy
         run: terraform destroy -auto-approve

.github/workflows/ipa-image-promote.yml

@@ -11,6 +11,14 @@ on:
         description: Promote Rocky Linux 9 aarch64
         type: boolean
         default: true
+      rocky10:
+        description: Promote Rocky Linux 10
+        type: boolean
+        default: true
+      rocky10-aarch64:
+        description: Promote Rocky Linux 10 aarch64
+        type: boolean
+        default: true
       ubuntu-noble:
         description: Promote Ubuntu 24.04 Noble
         type: boolean
@@ -29,7 +37,13 @@ jobs:
     steps:
       - name: Validate inputs
         run: |
-          if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.rocky9-aarch64 }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then
+          if [[
+            "${{ inputs.rocky9 }}" == "false" &&
+            "${{ inputs.rocky9-aarch64 }}" == "false" &&
+            "${{ inputs.rocky10 }}" == "false" &&
+            "${{ inputs.rocky10-aarch64 }}" == "false" &&
+            "${{ inputs.ubuntu-noble }}" == "false"
+          ]]; then
             echo "At least one distribution must be selected"
             exit 1
           fi
@@ -102,6 +116,33 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
         if: inputs.rocky9-aarch64
 
+      - name: Promote Rocky Linux 10 IPA image artifact
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-promote.yml \
+          -e artifact_type="ipa-images" \
+          -e os_distribution='rocky' \
+          -e os_release='10'
+        env:
+          ARTIFACT_TAG: ${{ inputs.image_tag }}
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10
+
+      - name: Promote Rocky Linux 10 aarch64 IPA image artifact
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-promote.yml \
+          -e repository_name="ipa-images-${{ steps.openstack_release.outputs.openstack_release }}-rocky-10-aarch64" \
+          -e pulp_base_path="ipa-images/${{ steps.openstack_release.outputs.openstack_release }}/rocky/10/aarch64"
+        env:
+          ARTIFACT_TAG: ${{ inputs.image_tag }}
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
+        if: inputs.rocky10-aarch64
+
       - name: Promote Ubuntu Noble 24.04 IPA image artifact
         run: |
           source venvs/kayobe/bin/activate &&

etc/kayobe/pulp-ipa-image-versions.yml

@@ -2,4 +2,6 @@
 # IPA image versioning tags
 stackhpc_rocky_9_ipa_image_version: "2025.1-20260420T095100"
 stackhpc_rocky_9_ipa_image_version_aarch64: "2025.1-20260420T095100"
+stackhpc_rocky_10_ipa_image_version: "2025.1-20260805T091024"
+stackhpc_rocky_10_ipa_image_version_aarch64: "2025.1-20260805T091024"
 stackhpc_ubuntu_noble_ipa_image_version: "2025.1-20260420T095100"

etc/kayobe/stackhpc-ipa-images.yml

@@ -12,20 +12,17 @@ stackhpc_ipa_image_overcloud_enabled: true
 # Options are "amd64" and "aarch64".
 stackhpc_ipa_arch: amd64
 
-# NOTE(owenjones): for now we'll override the release used so we use CentOS
-# Stream 9 IPA images on both Rocky 9 and 10 - remove once Rocky 10 IPA images
-# are stable.
-stackhpc_ipa_release: "{{ '9' if os_distribution == 'rocky' and os_release == '10' else os_release }}"
-
 # The IPA image source, defined by os_distribution,
 # os_release, and the current stable version.
 stackhpc_ipa_image_url: "{{ stackhpc_release_pulp_content_url }}/ipa-images/\
                          {{ openstack_release }}/{{ os_distribution }}/\
-                         {{ stackhpc_ipa_release }}{{ '/aarch64' if os_distribution == 'rocky' and stackhpc_ipa_arch == 'aarch64' else '' }}/\
+                         {{ os_release }}{{ '/aarch64' if os_distribution == 'rocky' and stackhpc_ipa_arch == 'aarch64' else '' }}/\
                          {{ stackhpc_ipa_image_version }}"
 
 # IPA image version tag selection
 stackhpc_ipa_image_version: >-
-  {{ stackhpc_rocky_9_ipa_image_version_aarch64 if os_distribution == 'rocky' and stackhpc_ipa_arch == 'aarch64' else
-  stackhpc_rocky_9_ipa_image_version if os_distribution == 'rocky' else
+  {{ stackhpc_rocky_9_ipa_image_version_aarch64 if os_distribution == 'rocky' and os_release == '9' and stackhpc_ipa_arch == 'aarch64' else
+  stackhpc_rocky_10_ipa_image_version_aarch64 if os_distribution == 'rocky' and os_release == '10' and stackhpc_ipa_arch == 'aarch64' else
+  stackhpc_rocky_9_ipa_image_version if os_distribution == 'rocky' and os_release == '9' else
+  stackhpc_rocky_10_ipa_image_version if os_distribution == 'rocky' and os_release == '10' else
   stackhpc_ubuntu_noble_ipa_image_version if os_distribution == 'ubuntu' and os_release == 'noble' }}

releasenotes/notes/centos-stream-10-ipa-dde5cf23f432b541.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Issues with CentOS Stream 10 IPA images have been resolved
+    upstream; these images are now buildable and used with Rocky 10
+    hosts.