diff --git a/.github/workflows/scripts/compare_to_generic.sh b/.github/workflows/scripts/compare_to_generic.sh index 8f046ec9d4..0febce8241 100755 --- a/.github/workflows/scripts/compare_to_generic.sh +++ b/.github/workflows/scripts/compare_to_generic.sh @@ -29,16 +29,17 @@ if ! python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_module exit 1 fi -# Also compare NVIDIA GPU software stacks -if [[ -n "$CUDA_COMPUTE_CAPABILITIES" ]]; then - read -ra compute_capabilities <<< "$CUDA_COMPUTE_CAPABILITIES" - echo "Also comparing CUDA-enabled software stacks (for compute capabilities: ${compute_capabilities[@]})" +# Also compare accelerator software stacks +if [[ -n "$ACCELERATOR_TARGETS" ]]; then + read -ra accel_capabilities <<< "$ACCELERATOR_TARGETS" + echo "Also comparing accelerator-enabled software stacks (for compute capabilities: ${accel_capabilities[@]})" # Initialize a variable to track failures any_failure=0 # Loop over the array - for cc in "${compute_capabilities[@]}"; do - source_of_truth_modules="$base_dir/$source_of_truth/accel/nvidia/cc80/$modules_subdir" - arch_modules="$base_dir/$target_arch/accel/nvidia/$cc/$modules_subdir" + for accel in "${accel_capabilities[@]}"; do + source_of_truth_accel="${ACCELERATOR_TARGETS%% *}" # Just use the first entry as source of truth + source_of_truth_modules="$base_dir/$source_of_truth/accel/${source_of_truth_accel}/$modules_subdir" + arch_modules="$base_dir/$target_arch/accel/$accel/$modules_subdir" echo "Comparing $arch_modules to $source_of_truth_modules" if ! python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_modules; then echo "Warning: Comparison failed for compute capability $cc" >&2 @@ -46,9 +47,9 @@ if [[ -n "$CUDA_COMPUTE_CAPABILITIES" ]]; then fi done if [[ $any_failure -ne 0 ]]; then - echo "One or more CUDA software stack comparisons failed." >&2 + echo "One or more accelerator software stack comparisons failed." >&2 exit 1 fi else - echo "CUDA_COMPUTE_CAPABILITIES is not set or is empty, not checking NVIDIA software stacks" + echo "ACCELERATOR_TARGETS is not set or is empty, not checking accelerator software stacks" fi diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e5611e95f3..1f4a087221 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -28,6 +28,7 @@ env: - aarch64/neoverse_v1 - aarch64/nvidia/grace - aarch64/a64fx + 2025.06: x86_64: - x86_64/amd/zen2 @@ -46,6 +47,27 @@ env: - aarch64/neoverse_v1 - aarch64/nvidia/grace - aarch64/a64fx + + 2026.06: + x86_64: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/amd/zen5 + - x86_64/intel/haswell + - x86_64/intel/sapphirerapids + - x86_64/intel/skylake_avx512 + - x86_64/intel/icelake + - x86_64/intel/cascadelake + - x86_64/generic + aarch64: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - aarch64/nvidia/grace + - aarch64/a64fx + - aarch64/aws/graviton4 + # list below should correspond with table @ https://eessi.io/docs/software_layer/gpu_targets/ EESSI_ACCELERATOR_TARGETS: | 2023.06: @@ -60,8 +82,8 @@ env: - nvidia/cc80 - nvidia/cc90 aarch64/a64fx: [] + 2025.06: - # Provide a default set of compute capabilities default: - nvidia/cc70 - nvidia/cc80 @@ -76,7 +98,47 @@ env: - amd/gfx1101 - amd/gfx1200 - amd/gfx1201 - # and then allow for special cases for specific architectures + aarch64/generic: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_n1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_v1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/nvidia/grace: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/a64fx: [] + + 2026.06: + default: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + - amd/gfx908 + - amd/gfx90a + - amd/gfx942 + - amd/gfx1030 + - amd/gfx1100 + - amd/gfx1101 + - amd/gfx1200 + - amd/gfx1201 aarch64/generic: - nvidia/cc70 - nvidia/cc80 @@ -148,6 +210,7 @@ jobs: echo "EOF" >> "$GITHUB_OUTPUT" requires_forced_ci: + if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 needs: check_EESSI_version_changed_files steps: @@ -180,15 +243,21 @@ jobs: # Arm CPU targets (EESSI 2023.06) - runs_on: ubuntu-24.04-arm EESSI_VERSION: 2023.06 - # Arm CPU targets (EESSI 2025.06) - - runs_on: ubuntu-24.04-arm - EESSI_VERSION: 2025.06 # x86_64 CPU targets (EESSI 2023.06) - runs_on: ubuntu-24.04 EESSI_VERSION: 2023.06 + # Arm CPU targets (EESSI 2025.06) + - runs_on: ubuntu-24.04-arm + EESSI_VERSION: 2025.06 # x86_64 CPU targets (EESSI 2025.06) - runs_on: ubuntu-24.04 EESSI_VERSION: 2025.06 + # Arm CPU targets (EESSI 2026.06) + - runs_on: ubuntu-24.04-arm + EESSI_VERSION: 2026.06 + # x86_64 CPU targets (EESSI 2026.06) + - runs_on: ubuntu-24.04 + EESSI_VERSION: 2026.06 runs-on: ${{ matrix.runs_on }} steps: @@ -209,12 +278,10 @@ jobs: echo "lscpu:" lscpu - - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@10197e000cc0add8e54ac4fb73d3ed44e2de72b4 # v5.5 + - name: Mount EESSI CernVM-FS repository + uses: eessi/github-action-eessi@v3 with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io + eessi_stack_version: '${{matrix.EESSI_VERSION}}' - name: Check for missing installations if: > @@ -224,9 +291,6 @@ jobs: matrix.EESSI_VERSION ) run: | - # Do a basic initialisation to set up Lmod and EESSI modules - export EESSI_MODULE_STICKY=1 - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/lmod/bash # From this extract the CPUs we need to loop over cpu_subdirs=$(echo "${EESSI_CPU_TARGETS}" | yq ".\"${EESSI_VERSION}\".\"${EESSI_CPU_FAMILY}\" | .[]") @@ -254,7 +318,7 @@ jobs: ( - set -euo pipefail + set -e -o pipefail module --force purge export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$cpu" diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml index 4e3a8db6f6..03a7a65679 100644 --- a/.github/workflows/test_compare_stacks.yml +++ b/.github/workflows/test_compare_stacks.yml @@ -1,89 +1,260 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Check for discrepancies between software stacks in software.eessi.io + on: - push: - branches: [ "main" ] - pull_request: + workflow_run: + workflows: + - Check for missing software installations in software.eessi.io + types: + - completed + workflow_dispatch: + permissions: - contents: read # to fetch code (actions/checkout) + contents: read + env: - CUDA_COMPUTE_CAPABILITIES_YAML: | + # list below should correspond with table @ https://eessi.io/docs/software_layer/cpu_targets/ + EESSI_CPU_TARGETS: | + 2023.06: + x86_64: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/intel/haswell + - x86_64/intel/sapphirerapids + - x86_64/intel/skylake_avx512 + - x86_64/intel/icelake + - x86_64/intel/cascadelake + - x86_64/generic + aarch64: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - aarch64/nvidia/grace + - aarch64/a64fx + + 2025.06: + x86_64: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/amd/zen5 + - x86_64/intel/haswell + - x86_64/intel/sapphirerapids + - x86_64/intel/skylake_avx512 + - x86_64/intel/icelake + - x86_64/intel/cascadelake + - x86_64/generic + aarch64: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - aarch64/nvidia/grace + - aarch64/a64fx + + 2026.06: + x86_64: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/amd/zen5 + - x86_64/intel/haswell + - x86_64/intel/sapphirerapids + - x86_64/intel/skylake_avx512 + - x86_64/intel/icelake + - x86_64/intel/cascadelake + - x86_64/generic + aarch64: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - aarch64/nvidia/grace + - aarch64/a64fx + - aarch64/aws/graviton4 + + # list below should correspond with table @ https://eessi.io/docs/software_layer/gpu_targets/ + EESSI_ACCELERATOR_TARGETS: | 2023.06: # Provide a default set of compute capabilities default: - - cc70 - - cc80 - - cc90 + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 # and then allow for special cases for specific architectures x86_64/amd/zen2: - - cc70 - - cc80 - - cc90 + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 aarch64/a64fx: [] + 2025.06: - # Provide a default set of compute capabilities default: - - cc70 - - cc80 - - cc90 - - cc100 - - cc120 + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + - amd/gfx908 + - amd/gfx90a + - amd/gfx942 + - amd/gfx1030 + - amd/gfx1100 + - amd/gfx1101 + - amd/gfx1200 + - amd/gfx1201 + aarch64/generic: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_n1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_v1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/nvidia/grace: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/a64fx: [] + + 2026.06: + default: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + - amd/gfx908 + - amd/gfx90a + - amd/gfx942 + - amd/gfx1030 + - amd/gfx1100 + - amd/gfx1101 + - amd/gfx1200 + - amd/gfx1201 + aarch64/generic: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_n1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/neoverse_v1: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 + aarch64/nvidia/grace: + - nvidia/cc70 + - nvidia/cc80 + - nvidia/cc90 + - nvidia/cc100 + - nvidia/cc120 aarch64/a64fx: [] jobs: compare_stacks: - runs-on: ubuntu-24.04 + # Only run after the missing-installation workflow succeeds. + # workflow_run itself is triggered for every completed run, but this + # prevents this job from running for failed/cancelled upstream runs. + if: > + github.event_name == 'workflow_dispatch' + || github.event.workflow_run.conclusion == 'success' + strategy: fail-fast: false matrix: - EESSI_VERSION: - - 2023.06 - - 2025.06 - - 2026.06 - COMPARISON_ARCH: - - aarch64/a64fx - - aarch64/aws/graviton4 - - aarch64/neoverse_n1 - - aarch64/neoverse_v1 - - aarch64/nvidia/grace - - x86_64/amd/zen2 - - x86_64/amd/zen3 - - x86_64/amd/zen4 - - x86_64/amd/zen5 - - x86_64/intel/haswell - - x86_64/intel/skylake_avx512 - - x86_64/intel/sapphirerapids - - x86_64/intel/icelake - - x86_64/intel/cascadelake - exclude: - # amd/zen5 only supported since EESSI 2025.06 - - EESSI_VERSION: 2023.06 - COMPARISON_ARCH: x86_64/amd/zen5 - # aws/graviton4 only supported since EESSI 2026.06 - - EESSI_VERSION: 2023.06 - COMPARISON_ARCH: aarch64/aws/graviton4 - - EESSI_VERSION: 2025.06 - COMPARISON_ARCH: aarch64/aws/graviton4 + include: + # Arm CPU targets (EESSI 2023.06) + - runs_on: ubuntu-24.04-arm + EESSI_VERSION: 2023.06 + # x86_64 CPU targets (EESSI 2023.06) + - runs_on: ubuntu-24.04 + EESSI_VERSION: 2023.06 + # Arm CPU targets (EESSI 2025.06) + - runs_on: ubuntu-24.04-arm + EESSI_VERSION: 2025.06 + # x86_64 CPU targets (EESSI 2025.06) + - runs_on: ubuntu-24.04 + EESSI_VERSION: 2025.06 + # Arm CPU targets (EESSI 2026.06) + - runs_on: ubuntu-24.04-arm + EESSI_VERSION: 2026.06 + # x86_64 CPU targets (EESSI 2026.06) + - runs_on: ubuntu-24.04 + EESSI_VERSION: 2026.06 + + runs-on: ${{ matrix.runs_on }} + steps: - - name: Check out software-layer repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Mount EESSI CernVM-FS pilot repository - uses: eessi/github-action-eessi@v3 - - - name: Compare stacks - run: | - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} - export EESSI_OS_TYPE=linux - env | grep ^EESSI | sort - - # Compare the requested architecture to the generic stack - # (assumes the general structure /cvmfs/software.eessi.io/versions/2023.06/software/linux/$COMPARISON_ARCH/modules/all) - # and include a check for CUDA-enabled software using the environment variable CUDA_COMPUTE_CAPABILITIES - # (which assumes the structure /cvmfs/software.eessi.io/versions/2023.06/software/linux/$COMPARISON_ARCH/accel/nvidia/$cc/modules/all) - - # Parse the yaml that makes the compute capabilities arch-dependent - CUDA_COMPUTE_CAPABILITIES=$(echo "${CUDA_COMPUTE_CAPABILITIES_YAML}" | yq ".\"${{matrix.EESSI_VERSION}}\".\"${{matrix.COMPARISON_ARCH}}\" // .\"${{matrix.EESSI_VERSION}}\".default | .[]" | tr '\n' ' ') - export CUDA_COMPUTE_CAPABILITIES=${CUDA_COMPUTE_CAPABILITIES%% } # trim trailing space - - .github/workflows/scripts/compare_to_generic.sh ${EESSI_PREFIX}/software/${EESSI_OS_TYPE} ${{matrix.COMPARISON_ARCH}} + - name: Check out software-layer repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # For workflow_run events, check the exact commit that was tested + # by the upstream workflow. + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + + - name: Mount EESSI CernVM-FS repository + uses: eessi/github-action-eessi@v3 + with: + eessi_stack_version: '${{matrix.EESSI_VERSION}}' + + - name: Compare stacks + shell: bash + run: | + echo "::group::EESSI ${EESSI_VERSION} / ${EESSI_CPU_FAMILY}" + + # Get all CPU targets belonging to this architecture family. + CPU_TARGETS=$(echo "${EESSI_CPU_TARGETS}" \ + | yq -r ".\"${EESSI_VERSION}\".\"${EESSI_CPU_FAMILY}\"[]") + + for COMPARISON_ARCH in ${CPU_TARGETS}; do + echo "::group::Compare ${EESSI_VERSION} / ${COMPARISON_ARCH}" + + # Use architecture-specific accelerator targets where defined, + # otherwise fall back to the default set. + ALL_ACCELERATOR_TARGETS=$(echo "${EESSI_ACCELERATOR_TARGETS}" \ + | yq ".\"${{ matrix.EESSI_VERSION }}\".\"${COMPARISON_ARCH}\" // .\"${{ matrix.EESSI_VERSION }}\".default | .[]" \ + | tr '\n' ' ') + + echo "COMPARISON_ARCH=${COMPARISON_ARCH}" + echo "ACCELERATOR_TARGETS=${ALL_ACCELERATOR_TARGETS}" + + for ACCELERATOR_TYPE in nvidia amd; do + ACCELERATOR_TARGETS_TYPE=$(echo "${ALL_ACCELERATOR_TARGETS}" \ + | tr ' ' '\n' \ + | grep "^${ACCELERATOR_TYPE}/" \ + | tr '\n' ' ' || true) + + if [[ -z "${ACCELERATOR_TARGETS_TYPE}" ]]; then + continue + fi + + export ACCELERATOR_TARGETS="${ACCELERATOR_TARGETS_TYPE}" + + echo "Comparing ${ACCELERATOR_TYPE} accelerator targets:" + echo "ACCELERATOR_TARGETS=${ACCELERATOR_TARGETS}" + + .github/workflows/scripts/compare_to_generic.sh \ + "${EESSI_PREFIX}/software/${EESSI_OS_TYPE}" \ + "${COMPARISON_ARCH}" + done + + echo "::endgroup::" + done