llext: fix auxiliary module ordering

Fully automated module dependency tree generation isn't planned for
SOF yet, so for now we do this manually. At the moment the sorting is
completely random, relying on Python's os.scandir() output. This
patch makes it possible to order auxiliary LLEXT modules explicitly.
It supports up to 9 dependency levels with level 1 being the most
shallow one, i.e. only end-user modules can depend on it. If any
level 1 modules depend on further modules, those are called "level 2"
modules etc. For this llext targets in respective CMakeLists.txt
files have to be called, e.g. "aux1_iir" - see existing examples.
This then guarantees correct ordering and successful run-time symbol
resolution.

Signed-off-by: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>

Author: lyakh

scripts/xtensa-build-zephyr.py

@@ -40,6 +40,7 @@
 import dataclasses
 import concurrent.futures as concurrent
 import tarfile
+import re
 
 from west import configuration as west_config
 
@@ -967,73 +968,77 @@ def install_lib(platform, sof_output_dir, abs_build_dir, platform_wconfig):
 
 	sof_lib_dir.mkdir(parents=True, exist_ok=True)
 
-	with os.scandir(str(abs_build_dir)) as iter:
-		for entry in iter:
-			if (not entry.is_dir or
-			    not entry.name.endswith('_llext')):
-				continue
-
-			entry_path = pathlib.Path(entry.path)
-
-			uuids = entry_path / 'llext.uuid'
-			if not os.path.exists(uuids):
-				print(f"Directory {entry.name} has no llext.uuid file. Skipping.")
-				continue
+	dirlist = [d for d in os.listdir(str(abs_build_dir)) if os.path.isdir(abs_build_dir / d)]
+	# Find all auxiliary modules, they're called "aux1_", "aux2_", etc.,
+	# where lower numbers mean fewer dependent levels, so we need to sort
+	# them in reversed order to have all dependencies satisfied
+	pattern_aux = re.compile(r'^aux\d_.+_llext$')
+	aux = sorted([d for d in dirlist if pattern_aux.match(d)], reverse=True)
+	llext_dirs = [d for d in dirlist if d.endswith('_llext') and d not in aux]
+	final_list = aux + llext_dirs
+
+	for entry in final_list:
+		entry_path = abs_build_dir / entry
+
+		uuids = entry_path / 'llext.uuid'
+		if not os.path.exists(uuids):
+			print(f"Directory {entry} has no llext.uuid file. Skipping.")
+			continue
 
-			# replace '_llext' with '.llext', e.g.
-			# eq_iir_llext/eq_iir.llext
-			llext_base = entry.name[:-6]
-			llext_file = llext_base + '.llext'
-			lib_name = ''
-
-			lib_fname = entry_path / 'lib_name.txt'
-			if os.path.exists(lib_fname):
-				with open(lib_fname, 'r') as libs_f:
-					lib_name = libs_f.read()
-					if lib_name not in libs.keys():
-						libs[lib_name] = []
-					libs[lib_name].append(str(entry_path / llext_file))
-			else:
-				dst = sof_lib_dir / llext_file
-
-				rimage_cfg = entry_path / 'rimage_config.toml'
-				llext_input = entry_path / (llext_base + '.llext')
-				llext_output = entry_path / (llext_file + '.ri')
-
-				# See why the shlex() parsing step is required at
-				# https://docs.zephyrproject.org/latest/develop/west/sign.html#rimage
-				# and in Zephyr commit 030b740bd1ec
-				sign_cmd = [rimage_cmd, "-o", str(llext_output),
-					    "-e", "-c", str(rimage_cfg),
-					    "-k", str(signing_key), "-l", "-r"]
-				if _ws_args is not None:
-					sign_cmd.extend(shlex.split(_ws_args))
-				sign_cmd.append(str(llext_input))
-				execute_command(sign_cmd, cwd=west_top)
-
-				# An intuitive way to make this multiline would be
-				# with (open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext,
-				#	open(llext_output.with_suffix('.llext.xman'), 'rb') as fman):
-				# but a Python version, used on Windows errored out on this.
-				# Thus we're left with a choice between a 150-character
-				# long line and an illogical split like this
-				with open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, open(
-					  llext_output.with_suffix('.ri.xman'), 'rb') as fman:
-					# Concatenate the manifest and the llext
-					shutil.copyfileobj(fman, fdst)
-					shutil.copyfileobj(fllext, fdst)
-
-			# Create symbolic links for all UUIDs
-			with open(uuids, 'r') as uuids_f:
-				for uuid in uuids_f:
-					if os.path.exists(lib_fname):
-						if lib_name not in lib_uuids.keys():
-							lib_uuids[lib_name] = []
-						lib_uuids[lib_name].append(uuid.strip())
-					else:
-						linkname = uuid.strip() + '.bin'
-						symlink_or_copy(sof_lib_dir, llext_file,
-								sof_lib_dir, linkname)
+		# replace '_llext' with '.llext', e.g.
+		# eq_iir_llext/eq_iir.llext
+		llext_base = entry[:-6]
+		llext_file = llext_base + '.llext'
+		lib_name = ''
+
+		lib_fname = entry_path / 'lib_name.txt'
+		if os.path.exists(lib_fname):
+			with open(lib_fname, 'r') as libs_f:
+				lib_name = libs_f.read()
+				if lib_name not in libs.keys():
+					libs[lib_name] = []
+				libs[lib_name].append(str(entry_path / llext_file))
+		else:
+			dst = sof_lib_dir / llext_file
+
+			rimage_cfg = entry_path / 'rimage_config.toml'
+			llext_input = entry_path / (llext_base + '.llext')
+			llext_output = entry_path / (llext_file + '.ri')
+
+			# See why the shlex() parsing step is required at
+			# https://docs.zephyrproject.org/latest/develop/west/sign.html#rimage
+			# and in Zephyr commit 030b740bd1ec
+			sign_cmd = [rimage_cmd, "-o", str(llext_output),
+				    "-e", "-c", str(rimage_cfg),
+				    "-k", str(signing_key), "-l", "-r"]
+			if _ws_args is not None:
+				sign_cmd.extend(shlex.split(_ws_args))
+			sign_cmd.append(str(llext_input))
+			execute_command(sign_cmd, cwd=west_top)
+
+			# An intuitive way to make this multiline would be
+			# with (open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext,
+			#	open(llext_output.with_suffix('.llext.xman'), 'rb') as fman):
+			# but a Python version, used on Windows errored out on this.
+			# Thus we're left with a choice between a 150-character
+			# long line and an illogical split like this
+			with open(dst, 'wb') as fdst, open(llext_output, 'rb') as fllext, open(
+				  llext_output.with_suffix('.ri.xman'), 'rb') as fman:
+				# Concatenate the manifest and the llext
+				shutil.copyfileobj(fman, fdst)
+				shutil.copyfileobj(fllext, fdst)
+
+		# Create symbolic links for all UUIDs
+		with open(uuids, 'r') as uuids_f:
+			for uuid in uuids_f:
+				if os.path.exists(lib_fname):
+					if lib_name not in lib_uuids.keys():
+						lib_uuids[lib_name] = []
+					lib_uuids[lib_name].append(uuid.strip())
+				else:
+					linkname = uuid.strip() + '.bin'
+					symlink_or_copy(sof_lib_dir, llext_file,
+							sof_lib_dir, linkname)
 
 	lib_install_dir = sof_output_dir / platform
 	if args.key_type_subdir != "none":

src/math/CMakeLists.txt

@@ -44,8 +44,8 @@ if(CONFIG_BINARY_LOGARITHM_FIXED)
 endif()
 
 if(CONFIG_MATH_FIR STREQUAL "m")
-  add_subdirectory(fir_llext ${PROJECT_BINARY_DIR}/fir_llext)
-  add_dependencies(app fir)
+  add_subdirectory(fir_llext ${PROJECT_BINARY_DIR}/aux1_fir_llext)
+  add_dependencies(app aux1_fir)
 elseif(CONFIG_MATH_FIR)
   list(APPEND base_files fir_generic.c fir_hifi2ep.c fir_hifi3.c fir_hifi5.c)
 endif()
@@ -55,8 +55,8 @@ if(CONFIG_MATH_FFT)
 endif()
 
 if(CONFIG_MATH_IIR STREQUAL "m")
-  add_subdirectory(iir_llext ${PROJECT_BINARY_DIR}/iir_llext)
-  add_dependencies(app iir)
+  add_subdirectory(iir_llext ${PROJECT_BINARY_DIR}/aux1_iir_llext)
+  add_dependencies(app aux1_iir)
 elseif(CONFIG_MATH_IIR)
   if(CONFIG_MATH_IIR_DF2T)
     list(APPEND base_files iir_df2t_generic.c iir_df2t_hifi3.c iir_df2t.c)

src/math/fir_llext/CMakeLists.txt

@@ -1,7 +1,7 @@
 # Copyright (c) 2024 Intel Corporation.
 # SPDX-License-Identifier: Apache-2.0
 
-sof_llext_build("fir"
+sof_llext_build("aux1_fir"
 	SOURCES fir_common.c
 		../fir_generic.c
 		../fir_hifi2ep.c

src/math/iir_llext/CMakeLists.txt

@@ -15,7 +15,7 @@ else()
 endif()
 
 if(CONFIG_MATH_IIR_DF1 OR CONFIG_MATH_IIR_DF2T)
-  sof_llext_build("iir"
+  sof_llext_build("aux1_iir"
 		  SOURCES iir.c ${df1} ${df2t}
 		  LIB openmodules
   )