Add support for RISC-V repository

.github/workflows/docs.yml

@@ -30,9 +30,8 @@ jobs:
       - name: Generate API data
         run: |
           echo "Generating data files..."
-          module purge
-          module unuse $MODULEPATH
-          module use /cvmfs/software.eessi.io/init/modules/
+          module --force purge
+
           # Set CPU override for the stack to query
           export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="x86_64/intel/icelake"
           # Set GPU overrides for the stack to query (we have no GPU, so also override the GPU check)
@@ -60,11 +59,38 @@ jobs:
           # Merge all these results together
           python scripts/merge_data_files.py out.yaml eessi*.yaml
           mv out.yaml docs/data/eessi_software_metadata.yaml
+
+          # Also do RISC-V repository
+          rm eessi*.yaml
+          # Need to set environment variables to load EESSI module for RISC-V
+          (
+            export EESSI_DEBUG_INIT=1
+            export EESSI_VERSION_OVERRIDE="2025.06-001"
+            export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic"
+            export EESSI_SOFTWARE_SUBDIR_OVERRIDE="riscv64/generic"
+            export EESSI_ACCELERATOR_TARGET_OVERRIDE="does/not/exist"
+            export EESSI_CPU_FAMILY_OVERRIDE="x86_64"
+            module load EESSI/2025.06
+            module load EasyBuild/5
+            module load EESSI-extend
+            echo $MODULEPATH
+            python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv"
+          )
+          python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml
+          mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml
+
           # Generate json data files and markdown index/description for them
           cd docs/data
           python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata
+          python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv
           python ../../scripts/calculate_hashes.py
-          for json_file in *.json; do
+          for json_file in eessi_api_metadata_*.json; do
+            python ../../scripts/generate_schema_md.py $json_file >> index.md
+          done
+          echo "" >> index.md
+          echo "### RISC-V datafiles"  >> index.md
+          echo "" >> index.md
+          for json_file in eessi_api_metadata-riscv_*.json; do
             python ../../scripts/generate_schema_md.py $json_file >> index.md
           done
       - run: |

.github/workflows/prs.yml

@@ -25,25 +25,58 @@ jobs:
       - name: Generate API data
         run: |
           echo "Generating data files..."
-          module purge
-          module unuse $MODULEPATH
-          module use /cvmfs/software.eessi.io/init/modules/
-          # Set CPU override for the stack to query
-          export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="x86_64/intel/icelake"
-          # Set GPU overrides for the stack to query (we have no GPU, so also override the GPU check)
-          export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90"
-          export EESSI_OVERRIDE_GPU_CHECK=1
+
           # Only do 2023.06 for EB 5 since this is just a test
-          ( module load EESSI/2023.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) &
+          (
+            module --force purge
+            export EESSI_DEBUG_INIT=1
+            # Set CPU override for the stack to query
+            export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="x86_64/intel/icelake"
+            # Set GPU overrides for the stack to query (we have no GPU, so also override the GPU check)
+            export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90"
+            export EESSI_OVERRIDE_GPU_CHECK=1
+            module load EESSI/2023.06
+            module load EasyBuild/5
+            module load EESSI-extend
+            python scripts/generate_data_files.py --eessi-version=2023.06
+          ) &
           # Merge all these results together
           wait
           python scripts/merge_data_files.py out.yaml eessi*.yaml
           mv out.yaml docs/data/eessi_software_metadata.yaml
+
+          # Also test RISC-V
+          rm eessi*.yaml
+          # Need to set environment variables to load EESSI module for RISC-V
+          (
+            module --force purge
+            export EESSI_DEBUG_INIT=1
+            export EESSI_VERSION_OVERRIDE="2025.06-001"
+            export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic"
+            export EESSI_SOFTWARE_SUBDIR_OVERRIDE="riscv64/generic"
+            export EESSI_ACCELERATOR_TARGET_OVERRIDE="does/not/exist"
+            export EESSI_CPU_FAMILY_OVERRIDE="x86_64"
+            module load EESSI/2025.06
+            module load EasyBuild/5
+            module load EESSI-extend
+            echo $MODULEPATH
+            python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv"
+          )
+          python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml
+          mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml
+
           # Generate json data files and markdown index/description for them
           cd docs/data
           python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata
+          python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv
           python ../../scripts/calculate_hashes.py
-          for json_file in *.json; do
+          for json_file in eessi_api_metadata_*.json; do
+            python ../../scripts/generate_schema_md.py $json_file >> index.md
+          done
+          echo "" >> index.md
+          echo "### RISC-V datafiles"  >> index.md
+          echo "" >> index.md
+          for json_file in eessi_api_metadata-riscv*.json; do
             python ../../scripts/generate_schema_md.py $json_file >> index.md
           done
       - name: Test building the website
@@ -54,4 +87,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: eessi-api-metadata
-          path: docs/data/eessi_api_metadata_software.json
+          path: docs/data/eessi_api_metadata*software*.json

scripts/generate_data_files.py

@@ -18,7 +18,16 @@
 from easybuild.tools.include import include_easyblocks
 from contextlib import contextmanager
 
-VALID_EESSI_VERSIONS = ["2025.06", "2023.06"]
+SUPPORTED_REPOSITORIES = {
+        'software.eessi.io': ["2025.06", "2023.06"],
+        'dev.eessi.io/riscv': ["2025.06-001"],
+}
+
+VALID_EESSI_VERSIONS = [
+    version
+    for versions in SUPPORTED_REPOSITORIES.values()
+    for version in versions
+]
 
 # Give order to my toolchains so I can easily figure out what "latest" means
 EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = OrderedDict(
@@ -67,7 +76,17 @@ def load_and_list_modules(full_module_name):
     """
 
     # Run as one shell script so the same session is used
+    # We first do a load attempt and catch any errors, if that works
+    # then do it for real
     cmd = f"""
+        out=$(module load {full_module_name} 2>&1)
+        status=$?
+
+        if [ $status -ne 0 ]; then
+            echo "$out" >&2
+            echo
+            exit $status
+        fi
         module load {full_module_name} >/dev/null 2>&1 || exit 1
         module --terse list 2>&1
     """
@@ -205,20 +224,31 @@ def merge_dicts(d1, d2):
         "-e",
         required=True,
         choices=VALID_EESSI_VERSIONS,
-        help=f"Allowed versions: {', '.join(VALID_EESSI_VERSIONS)}",
+        help=f"Allowed versions (also dependent on repository): {', '.join(VALID_EESSI_VERSIONS)}",
+    )
+    parser.add_argument(
+        "--repository",
+        "-r",
+        default="software.eessi.io",
+        choices=SUPPORTED_REPOSITORIES.keys(),
+        help=f"Repository to scan: {', '.join(SUPPORTED_REPOSITORIES)} (default: %(default)s)",
     )
 
     args = parser.parse_args()
     eessi_version = args.eessi_version
+    repository = args.repository
+
+    if eessi_version not in SUPPORTED_REPOSITORIES[repository]:
+        raise ValueError(f"You must choose an EESSI version supported by the repository: {SUPPORTED_REPOSITORIES[repository]}")
 
     print(f"Using EESSI version: {eessi_version}")
 
     # We use a single architecture path to gather information about the software versions
     eessi_reference_architecture = os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE", False)
     if not eessi_reference_architecture:
-        print("You must have selected a CPU architecture via EESSI_ARCHDETECT_OPTIONS_OVERRIDE")
+        print("You must have selected a CPU architecture via EESSI_ARCHDETECT_OPTIONS_OVERRIDE environment variable")
         exit()
-    base_path = f"/cvmfs/software.eessi.io/versions/{eessi_version}/software/linux/{eessi_reference_architecture}"
+    base_path = f"/cvmfs/{repository}/versions/{eessi_version}/software/linux/{eessi_reference_architecture}"
     cpu_easyconfig_files_dict = collect_eb_files(os.path.join(base_path, "software"))
     # We also gather all the acclerator installations for NVIDIA-enabled packages
     # We're not typically running this script on a node with a GPU so an override must have been set
@@ -243,7 +273,8 @@ def merge_dicts(d1, d2):
 
     # Store the toolchain hierarchies supported by the EESSI version
     eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"] = {}
-    for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version]:
+    # RISC-V versions have a stub like -001 at the end, make sure to drop it
+    for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version.split("-")[0]]:
         # versions are typically 2024a/2024b etc. for top level toolchains
         # so let's use that to make sorting easy
         toolchain_family = f"{top_level_toolchain['version']}_{top_level_toolchain['name']}"

scripts/process_eessi_software_metadata.py

@@ -24,6 +24,10 @@
     "x86_64/intel/cascadelake",
 ]
 
+RISCV_ARCHITECTURES = [
+    "riscv64/generic",
+]
+
 NVIDIA_ARCHITECTURES = [
     "accel/nvidia/cc70",
     "accel/nvidia/cc80",
@@ -63,7 +67,11 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool
     # 1) Detect the architecture substring inside the path
     base_version_dict["cpu_arch"] = []
     detected_arch = None
-    for arch in ARCHITECTURES:
+    if '/riscv64/' in original_path:
+        architecture_group = RISCV_ARCHITECTURES
+    else:
+        architecture_group = ARCHITECTURES
+    for arch in architecture_group:
         if f"/{arch}/" in original_path:
             detected_arch = arch
             break
@@ -89,11 +97,14 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool
     spider_cache = before_arch + detected_arch + "/.lmod/cache/spiderT.lua"
 
     # 3) Substitute each architecture and test module file existence in spider cache
-    for arch in ARCHITECTURES:
+    for arch in architecture_group:
         substituted_modulefile = modulefile.replace(detected_arch, arch)
         substituted_spider_cache = spider_cache.replace(detected_arch, arch)
-        # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache
-        found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0
+        # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can)
+        if os.path.exists(substituted_spider_cache):
+            found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0
+        else:
+            found = os.path.exists(substituted_modulefile)
         if found:
             base_version_dict["cpu_arch"].append(arch)
             # If we have an accelerator module let's check which architectures are supported
@@ -106,7 +117,15 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool
                         # Let's not include the "accel/" part of the accel_arch 
                         base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1))
                     else:
-                        print(f"No module {accel_substituted_modulefile}...not adding software for architecture {arch}/{accel_arch}")
+                        # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06
+                        if not (
+                            accel_substituted_modulefile.startswith('/cvmfs/software.eessi.io/versions/2023.06')
+                            and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"]
+                        ):
+                            print(
+                                f"No module {accel_substituted_modulefile}... "
+                                f"not adding software for architecture {arch}/{accel_arch}"
+                            )
                         continue
         else:
             print(f"No module {substituted_modulefile}...not adding software for architecture {arch}")
@@ -368,7 +387,7 @@ def main():
     base_json_metadata["architectures_map"] = {}
     for eessi_version in eessi_versions:
         base_json_metadata["architectures_map"][eessi_version] = {}
-        for architecture in ARCHITECTURES:
+        for architecture in ARCHITECTURES + RISCV_ARCHITECTURES:
             base_json_metadata["architectures_map"][eessi_version][architecture] = architecture
     base_json_metadata["gpu_architectures_map"] = {}
     base_json_metadata["category_details"] = {}