From 5104009ee4a9ff4a0568af02592ddcfcf52c9bca Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 19:33:21 -0700 Subject: [PATCH 01/21] Use meson as build system --- conda-recipe-cf/meta.yaml | 4 +- conda-recipe/meta.yaml | 4 +- meson.build | 64 +++++++++++ mkl/{_mkl_service.pyx => _py_mkl_service.pyx} | 0 pyproject.toml | 14 +-- setup.py | 102 ------------------ 6 files changed, 73 insertions(+), 115 deletions(-) create mode 100644 meson.build rename mkl/{_mkl_service.pyx => _py_mkl_service.pyx} (100%) delete mode 100644 setup.py diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml index d1f237d..42bd177 100644 --- a/conda-recipe-cf/meta.yaml +++ b/conda-recipe-cf/meta.yaml @@ -19,10 +19,12 @@ requirements: - {{ compiler('c') }} - {{ stdlib('c') }} host: + - meson-python >=0.13.0 + - meson + - pkg-config - python - python-gil # [py>=314] - pip >=25.0 - - setuptools >=77 - mkl-devel # [not osx] - mkl-devel <2024 # [osx] - cython diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 1ccfc05..e54b8df 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -18,10 +18,12 @@ requirements: - {{ compiler('c') }} - {{ stdlib('c') }} host: + - meson-python >=0.13.0 + - meson + - pkg-config - python - python-gil # [py>=314] - pip >=25.0 - - setuptools >=77 - mkl-devel - cython - wheel >=0.45.1 diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..1c9f560 --- /dev/null +++ b/meson.build @@ -0,0 +1,64 @@ +project( + 'mkl-service', + ['c', 'cython'], + version: run_command( + 'python', '-c', + 'import os; exec(open("mkl/_version.py").read()); print(__version__)', + check: true + ).stdout().strip(), + default_options: [ + 'buildtype=release', + ] +) + +py = import('python').find_installation(pure: false) + +c_args = ['-DNDEBUG'] + +thread_dep = dependency('threads') + +cc = meson.get_compiler('c') +mkl_rt = cc.find_library('mkl_rt', required: true) +mkl_dep = declare_dependency(dependencies: [mkl_rt]) + +rpath = '' +if host_machine.system() != 'windows' + rpath = '$ORIGIN/../..:$ORIGIN/../../..' +endif + +# C extension +py.extension_module( + '_mklinit', + sources: ['mkl/_mklinitmodule.c'], + dependencies: [mkl_dep, thread_dep], + c_args: c_args + ['-DUSING_MKL_RT'], + install_rpath: rpath, + install: true, + subdir: 'mkl' +) + +# Cython extension +py.extension_module( + '_py_mkl_service', + sources: ['mkl/_py_mkl_service.pyx'], + dependencies: [mkl_dep], + c_args: c_args, + install_rpath: rpath, + install: true, + subdir: 'mkl' +) + +# Python sources +py.install_sources( + [ + 'mkl/__init__.py', + 'mkl/_init_helper.py', + 'mkl/_version.py', + ], + subdir: 'mkl' +) + +install_subdir( + 'mkl/tests', + install_dir: py.get_install_dir() / 'mkl' +) diff --git a/mkl/_mkl_service.pyx b/mkl/_py_mkl_service.pyx similarity index 100% rename from mkl/_mkl_service.pyx rename to mkl/_py_mkl_service.pyx diff --git a/pyproject.toml b/pyproject.toml index 4f2a1b4..8f143f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,19 +24,11 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. [build-system] -build-backend = "setuptools.build_meta" -requires = [ - "setuptools>=77", - "Cython", - "wheel>=0.45.1", - "build>=1.2.2", - "mkl-devel" -] +build-backend = "mesonpy" +requires = ["meson-python>=0.13.0", "Cython", "mkl-devel"] [project] -authors = [ - {name = "Intel Corporation", email = "scripting@intel.com"} -] +authors = [{name = "Intel Corporation"}] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", diff --git a/setup.py b/setup.py deleted file mode 100644 index 0362197..0000000 --- a/setup.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2018, Intel Corporation -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Intel Corporation nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import os -import sys -from os.path import join - -import Cython.Build -from setuptools import Extension, setup - - -def extensions(): - mkl_root = os.environ.get("MKLROOT", None) - if mkl_root: - mkl_info = { - "include_dirs": [join(mkl_root, "include")], - "library_dirs": [ - join(mkl_root, "lib"), - join(mkl_root, "lib", "intel64"), - ], - "libraries": ["mkl_rt"], - } - else: - raise ValueError("MKLROOT environment variable not set.") - - if sys.platform != "win32": - mkl_info["rpaths"] = ["$ORIGIN/../..", "$ORIGIN/../../.."] - - mkl_include_dirs = mkl_info.get("include_dirs", []) - mkl_library_dirs = mkl_info.get("library_dirs", []) - mkl_libraries = mkl_info.get("libraries", ["mkl_rt"]) - mkl_rpaths = mkl_info.get("rpaths", []) - - defs = [] - if any(["mkl_rt" in li for li in mkl_libraries]): - # libs += ["dl"] - by default on Linux - defs += [("USING_MKL_RT", None)] - - extensions = [] - extensions.append( - Extension( - "mkl._mklinit", - sources=[join("mkl", "_mklinitmodule.c")], - include_dirs=mkl_include_dirs, - libraries=mkl_libraries - + (["pthread"] if os.name == "posix" else []), - library_dirs=mkl_library_dirs, - runtime_library_dirs=mkl_rpaths, - extra_compile_args=[ - "-DNDEBUG" - # "-g", "-O2", "-Wall", - ], - define_macros=defs, - ) - ) - - extensions.append( - Extension( - "mkl._py_mkl_service", - sources=[join("mkl", "_mkl_service.pyx")], - include_dirs=mkl_include_dirs, - library_dirs=mkl_library_dirs, - libraries=mkl_libraries, - runtime_library_dirs=mkl_rpaths, - extra_compile_args=[ - "-DNDEBUG" - # "-g", "-O2", "-Wall", - ], - ) - ) - - return extensions - - -setup( - cmdclass={"build_ext": Cython.Build.build_ext}, - ext_modules=extensions(), - zip_safe=False, -) From 4eb8b8efa165dcadf3922a5ce16f2fc78084dcb4 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 20:53:51 -0700 Subject: [PATCH 02/21] remove MKLROOT --- conda-recipe-cf/bld.bat | 1 - conda-recipe-cf/build.sh | 2 +- conda-recipe/bld.bat | 2 -- conda-recipe/build.sh | 2 -- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat index cc55004..ac9be74 100644 --- a/conda-recipe-cf/bld.bat +++ b/conda-recipe-cf/bld.bat @@ -1,5 +1,4 @@ @rem Remember to activate Intel Compiler, or remove these two lines to use Microsoft Visual Studio compiler -set MKLROOT=%PREFIX% %PYTHON% setup.py build --force install --old-and-unmanageable if errorlevel 1 exit 1 diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh index 5d36203..c81afab 100644 --- a/conda-recipe-cf/build.sh +++ b/conda-recipe-cf/build.sh @@ -1,2 +1,2 @@ #!/bin/bash -x -MKLROOT=$PREFIX $PYTHON setup.py build --force install --old-and-unmanageable +$PYTHON setup.py build --force install --old-and-unmanageable diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 90a55f4..ec28ad8 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -2,8 +2,6 @@ echo on rem set CFLAGS=-I%PREFIX%\Library\include %CFLAGS% rem set LDFLAGS=/LIBPATH:%PREFIX% %LDFLAGS% -set MKLROOT=%CONDA_PREFIX% - "%PYTHON%" setup.py clean --all :: Make CMake verbose diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 6bfa533..1b7067f 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,8 +1,6 @@ #!/bin/bash set -ex -export MKLROOT=$CONDA_PREFIX - read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" From ff78c6bfa093c489dea98b1b0d3030010e7b2d3b Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 21:02:42 -0700 Subject: [PATCH 03/21] add standard clang workflow and update clang workflow --- .github/workflows/build-with-clang.yml | 14 ++-- .../workflows/build-with-standard-clang.yml | 64 +++++++++++++++++++ 2 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/build-with-standard-clang.yml diff --git a/.github/workflows/build-with-clang.yml b/.github/workflows/build-with-clang.yml index 2c115d6..5c47b19 100644 --- a/.github/workflows/build-with-clang.yml +++ b/.github/workflows/build-with-clang.yml @@ -15,6 +15,7 @@ jobs: strategy: matrix: python: ["3.10", "3.11", "3.12", "3.13"] + numpy_version: ["numpy'>=2'"] env: ONEAPI_ROOT: /opt/intel/oneapi @@ -40,7 +41,6 @@ jobs: - name: Install Intel OneAPI run: | sudo apt-get install intel-oneapi-compiler-dpcpp-cpp - sudo apt-get install intel-oneapi-tbb sudo apt-get install intel-oneapi-mkl-devel - name: Setup Python @@ -55,12 +55,9 @@ jobs: fetch-depth: 0 - name: Install mkl-service dependencies - uses: BSFishy/pip-action@8f2d471d809dc20b6ada98c91910b6ae6243f318 # v1 - with: - packages: | - cython - setuptools>=77 - pytest + run: | + pip install meson-python cython mkl + pip install ${{ matrix.numpy_version }} - name: List oneAPI folder content run: ls ${{ env.ONEAPI_ROOT }}/compiler @@ -71,9 +68,10 @@ jobs: echo "$CMPLR_ROOT" export CC="$CMPLR_ROOT/bin/icx" export CFLAGS="${CFLAGS} -fno-fast-math" - python setup.py develop + pip install . --no-build-isolation --no-deps --verbose - name: Run mkl-service tests run: | source ${{ env.ONEAPI_ROOT }}/setvars.sh + pip install pytest pytest -s -v --pyargs mkl diff --git a/.github/workflows/build-with-standard-clang.yml b/.github/workflows/build-with-standard-clang.yml new file mode 100644 index 0000000..a91bc80 --- /dev/null +++ b/.github/workflows/build-with-standard-clang.yml @@ -0,0 +1,64 @@ +name: Build project with standard clang compiler + +on: + pull_request: + push: + branches: [master] + +permissions: read-all + +jobs: + build-with-standard-clang: + runs-on: ubuntu-latest + + strategy: + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + numpy_version: ["numpy'>=2'"] + + env: + COMPILER_ROOT: /usr/bin + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@3155a141048f8f89c06b4cdae32e7853e97536bc # 0.13.0 + with: + access_token: ${{ github.token }} + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python }} + architecture: x64 + + - name: Checkout repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Install mkl-service dependencies + run: | + pip install meson-python cython mkl + pip install ${{ matrix.numpy_version }} + + - name: Build mkl-service + run: | + export CC=${{ env.COMPILER_ROOT }}/clang + pip install . --no-build-isolation --no-deps --verbose + + - name: Run mkl-service tests + run: | + pip install pytest + # mkl-service cannot be installed in editable mode, we need + # to change directory before importing it and running tests + cd .. + pytest -s -v --pyargs mkl From 150ed1dac31a254e29f531e354dc60b09fa036e3 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 22:40:14 -0700 Subject: [PATCH 04/21] add ninja to build-system --- pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f143f1..431da5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,12 @@ [build-system] build-backend = "mesonpy" -requires = ["meson-python>=0.13.0", "Cython", "mkl-devel"] +requires = [ + "meson-python>=0.13.0", + "ninja", + "Cython", + "mkl-devel" +] [project] authors = [{name = "Intel Corporation"}] From 56fd3fb8126dd1189fd4e3cf9d08e73e2ee7dfc7 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 22:47:15 -0700 Subject: [PATCH 05/21] use cmake to find MKL and update dependencies --- .github/workflows/build-with-clang.yml | 2 +- .github/workflows/build-with-standard-clang.yml | 2 +- meson.build | 10 ++++++++-- pyproject.toml | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-with-clang.yml b/.github/workflows/build-with-clang.yml index 5c47b19..258bdbe 100644 --- a/.github/workflows/build-with-clang.yml +++ b/.github/workflows/build-with-clang.yml @@ -56,7 +56,7 @@ jobs: - name: Install mkl-service dependencies run: | - pip install meson-python cython mkl + pip install meson-python cython cmake ninja mkl pip install ${{ matrix.numpy_version }} - name: List oneAPI folder content diff --git a/.github/workflows/build-with-standard-clang.yml b/.github/workflows/build-with-standard-clang.yml index a91bc80..f100d32 100644 --- a/.github/workflows/build-with-standard-clang.yml +++ b/.github/workflows/build-with-standard-clang.yml @@ -47,7 +47,7 @@ jobs: - name: Install mkl-service dependencies run: | - pip install meson-python cython mkl + pip install meson-python cython cmake ninja mkl pip install ${{ matrix.numpy_version }} - name: Build mkl-service diff --git a/meson.build b/meson.build index 1c9f560..5e48d8b 100644 --- a/meson.build +++ b/meson.build @@ -18,8 +18,14 @@ c_args = ['-DNDEBUG'] thread_dep = dependency('threads') cc = meson.get_compiler('c') -mkl_rt = cc.find_library('mkl_rt', required: true) -mkl_dep = declare_dependency(dependencies: [mkl_rt]) +mkl_dep = dependency('MKL', method: 'cmake', + modules: ['MKL::MKL'], + cmake_args: [ + '-DMKL_ARCH=intel64', + '-DMKL_LINK=sdl', + ], + required: true +) rpath = '' if host_machine.system() != 'windows' diff --git a/pyproject.toml b/pyproject.toml index 431da5e..b153f74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ requires = [ "meson-python>=0.13.0", "ninja", "Cython", - "mkl-devel" + "mkl-devel", + "cmake" ] [project] From 41860b8e436df69335ec698253d99a90b55fb990 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 22:47:30 -0700 Subject: [PATCH 06/21] update meta.yamls and conda-forge conda recipes --- conda-recipe-cf/bld.bat | 2 +- conda-recipe-cf/build.sh | 2 +- conda-recipe-cf/meta.yaml | 4 ++-- conda-recipe/meta.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat index ac9be74..498077f 100644 --- a/conda-recipe-cf/bld.bat +++ b/conda-recipe-cf/bld.bat @@ -1,4 +1,4 @@ @rem Remember to activate Intel Compiler, or remove these two lines to use Microsoft Visual Studio compiler -%PYTHON% setup.py build --force install --old-and-unmanageable +%PYTHON% -m pip install --no-deps --no-build-isolation . if errorlevel 1 exit 1 diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh index c81afab..40c3b81 100644 --- a/conda-recipe-cf/build.sh +++ b/conda-recipe-cf/build.sh @@ -1,2 +1,2 @@ #!/bin/bash -x -$PYTHON setup.py build --force install --old-and-unmanageable +$PYTHON -m pip install --no-deps --no-build-isolation . diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml index 42bd177..2ce871e 100644 --- a/conda-recipe-cf/meta.yaml +++ b/conda-recipe-cf/meta.yaml @@ -21,7 +21,8 @@ requirements: host: - meson-python >=0.13.0 - meson - - pkg-config + - cmake + - ninja - python - python-gil # [py>=314] - pip >=25.0 @@ -29,7 +30,6 @@ requirements: - mkl-devel <2024 # [osx] - cython - wheel >=0.45.1 - - python-build >=1.2.2 run: - python - python-gil # [py>=314] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e54b8df..c2bfb03 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -20,14 +20,14 @@ requirements: host: - meson-python >=0.13.0 - meson - - pkg-config + - cmake + - ninja - python - python-gil # [py>=314] - pip >=25.0 - mkl-devel - cython - wheel >=0.45.1 - - python-build >=1.2.2 run: - python - python-gil # [py>=314] From de533655419944ff979be611a964751333dfd8f1 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 22:59:46 -0700 Subject: [PATCH 07/21] install mkl-devel when building with standard clang --- .github/workflows/build-with-standard-clang.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-with-standard-clang.yml b/.github/workflows/build-with-standard-clang.yml index f100d32..221188a 100644 --- a/.github/workflows/build-with-standard-clang.yml +++ b/.github/workflows/build-with-standard-clang.yml @@ -47,7 +47,7 @@ jobs: - name: Install mkl-service dependencies run: | - pip install meson-python cython cmake ninja mkl + pip install meson-python cython cmake ninja mkl-devel mkl pip install ${{ matrix.numpy_version }} - name: Build mkl-service From 580325028d8f48ef4ad801c73875742d094f159d Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 23:02:19 -0700 Subject: [PATCH 08/21] add pip build workflow --- .github/workflows/build_pip.yml | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/build_pip.yml diff --git a/.github/workflows/build_pip.yml b/.github/workflows/build_pip.yml new file mode 100644 index 0000000..cc9f24b --- /dev/null +++ b/.github/workflows/build_pip.yml @@ -0,0 +1,50 @@ +name: Editable build using pip and pre-release NumPy + +on: + push: + branches: + - master + pull_request: + +permissions: read-all + +env: + PACKAGE_NAME: mkl-service + MODULE_NAME: mkl-service + TEST_ENV_NAME: test_mkl_service + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + strategy: + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + use_pre: ["", "--pre"] + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0 + with: + miniforge-version: latest + channels: conda-forge + activate-environment: test + python-version: ${{ matrix.python }} + + - name: Install MKL + run: | + conda install mkl-devel mkl + + - name: Build conda package + run: | + pip install --no-cache-dir meson-python ninja cmake cython + pip install --no-cache-dir numpy ${{ matrix.use_pre }} + pip install -e ".[test]" --no-build-isolation --verbose + pip list + python -m pytest -v mkl/tests From fc310c7a518bf381f7f133d6ed44d553fd5d82ef Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 7 Apr 2026 23:35:31 -0700 Subject: [PATCH 09/21] update conda-recipe build scripts --- conda-recipe/bld.bat | 5 ----- conda-recipe/build.sh | 7 +++---- conda-recipe/meta.yaml | 1 + 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index ec28ad8..554e8f2 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -2,11 +2,6 @@ echo on rem set CFLAGS=-I%PREFIX%\Library\include %CFLAGS% rem set LDFLAGS=/LIBPATH:%PREFIX% %LDFLAGS% -"%PYTHON%" setup.py clean --all - -:: Make CMake verbose -set "VERBOSE=1" - :: -wnx flags mean: --wheel --no-isolation --skip-dependency-check %PYTHON% -m build -w -n -x if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 1b7067f..a18f7a9 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -4,10 +4,9 @@ set -ex read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" -${PYTHON} setup.py clean --all - -# Make CMake verbose -export VERBOSE=1 +if [ -d "build" ]; then + rm -rf build +fi # -wnx flags mean: --wheel --no-isolation --skip-dependency-check ${PYTHON} -m build -w -n -x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index c2bfb03..577c792 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,6 +24,7 @@ requirements: - ninja - python - python-gil # [py>=314] + - python-build - pip >=25.0 - mkl-devel - cython From 293dad4726bce458a015ee30a7591e1e98483e16 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 26 Apr 2026 02:44:35 -0700 Subject: [PATCH 10/21] address review feedback add build instructions --- README.md | 25 +++++++++++++++++++++++++ meson.build | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c397d0f..d1c603e 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,28 @@ A short example, illustrating its use: ``` For more information about the usage of support functions see [Developer Reference for Intel® oneAPI Math Kernel Library for C](https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-c/2025-2/support-functions.html). + +--- + +## Building + +A C compiler and Intel(R) OneAPI Math Kernel Library (OneMKL) are required to build mkl-service from source. + +Executing +```sh +python -m pip install . +``` + +will pull in the required build and runtime dependencies, including `mkl`, and build `mkl-service`. + +With an existing system or Conda `mkl` installation, build dependencies +- `mkl-devel` +- `meson-python` +- `cmake` +- `ninja` +- `cython` + +then, simply execute +``` +python -m pip install --no-build-isolation --no-deps . +``` diff --git a/meson.build b/meson.build index 5e48d8b..1ea4a7f 100644 --- a/meson.build +++ b/meson.build @@ -28,7 +28,7 @@ mkl_dep = dependency('MKL', method: 'cmake', ) rpath = '' -if host_machine.system() != 'windows' +if host_machine.system() == 'linux' rpath = '$ORIGIN/../..:$ORIGIN/../../..' endif From 69a56a81a36755df932c68693a2bdaee2705d148 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 26 Apr 2026 02:44:57 -0700 Subject: [PATCH 11/21] update workflows remove unnecessary packages and add python 3.14 to clang build --- .github/workflows/build-with-clang.yml | 4 ++-- .github/workflows/build-with-standard-clang.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-with-clang.yml b/.github/workflows/build-with-clang.yml index 258bdbe..c928fb8 100644 --- a/.github/workflows/build-with-clang.yml +++ b/.github/workflows/build-with-clang.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: - python: ["3.10", "3.11", "3.12", "3.13"] + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] numpy_version: ["numpy'>=2'"] env: @@ -56,7 +56,7 @@ jobs: - name: Install mkl-service dependencies run: | - pip install meson-python cython cmake ninja mkl + pip install meson-python cython cmake ninja pip install ${{ matrix.numpy_version }} - name: List oneAPI folder content diff --git a/.github/workflows/build-with-standard-clang.yml b/.github/workflows/build-with-standard-clang.yml index 221188a..983837e 100644 --- a/.github/workflows/build-with-standard-clang.yml +++ b/.github/workflows/build-with-standard-clang.yml @@ -47,7 +47,7 @@ jobs: - name: Install mkl-service dependencies run: | - pip install meson-python cython cmake ninja mkl-devel mkl + pip install meson-python cython cmake ninja mkl-devel pip install ${{ matrix.numpy_version }} - name: Build mkl-service From 8c15942b263e56e9e6cf9d31c7532d46cadb29a4 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 26 Apr 2026 03:33:12 -0700 Subject: [PATCH 12/21] remove copy-paste error numpy installations in workflows --- .github/workflows/build-with-clang.yml | 2 -- .github/workflows/build-with-standard-clang.yml | 2 -- .github/workflows/build_pip.yml | 4 +--- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build-with-clang.yml b/.github/workflows/build-with-clang.yml index c928fb8..2b3729c 100644 --- a/.github/workflows/build-with-clang.yml +++ b/.github/workflows/build-with-clang.yml @@ -15,7 +15,6 @@ jobs: strategy: matrix: python: ["3.10", "3.11", "3.12", "3.13", "3.14"] - numpy_version: ["numpy'>=2'"] env: ONEAPI_ROOT: /opt/intel/oneapi @@ -57,7 +56,6 @@ jobs: - name: Install mkl-service dependencies run: | pip install meson-python cython cmake ninja - pip install ${{ matrix.numpy_version }} - name: List oneAPI folder content run: ls ${{ env.ONEAPI_ROOT }}/compiler diff --git a/.github/workflows/build-with-standard-clang.yml b/.github/workflows/build-with-standard-clang.yml index 983837e..4652f89 100644 --- a/.github/workflows/build-with-standard-clang.yml +++ b/.github/workflows/build-with-standard-clang.yml @@ -14,7 +14,6 @@ jobs: strategy: matrix: python: ["3.10", "3.11", "3.12", "3.13", "3.14"] - numpy_version: ["numpy'>=2'"] env: COMPILER_ROOT: /usr/bin @@ -48,7 +47,6 @@ jobs: - name: Install mkl-service dependencies run: | pip install meson-python cython cmake ninja mkl-devel - pip install ${{ matrix.numpy_version }} - name: Build mkl-service run: | diff --git a/.github/workflows/build_pip.yml b/.github/workflows/build_pip.yml index cc9f24b..1a21d61 100644 --- a/.github/workflows/build_pip.yml +++ b/.github/workflows/build_pip.yml @@ -1,4 +1,4 @@ -name: Editable build using pip and pre-release NumPy +name: Editable build using pip on: push: @@ -23,7 +23,6 @@ jobs: strategy: matrix: python: ["3.10", "3.11", "3.12", "3.13", "3.14"] - use_pre: ["", "--pre"] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -44,7 +43,6 @@ jobs: - name: Build conda package run: | pip install --no-cache-dir meson-python ninja cmake cython - pip install --no-cache-dir numpy ${{ matrix.use_pre }} pip install -e ".[test]" --no-build-isolation --verbose pip list python -m pytest -v mkl/tests From ac6e9fce8c015b749aefecdd782d1103bd4c513c Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 26 Apr 2026 03:39:34 -0700 Subject: [PATCH 13/21] update agent instructions --- .github/AGENTS.md | 3 +++ .github/copilot-instructions.md | 2 +- AGENTS.md | 16 ++++++++++++---- mkl/AGENTS.md | 6 +++--- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/AGENTS.md b/.github/AGENTS.md index a318f75..a1041aa 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -4,7 +4,10 @@ CI/CD workflows, automation, security scanning, and package distribution. ## Workflows - **conda-package.yml** — main build/test pipeline (Linux/Windows, Python 3.10-3.14) +- **conda-package-cf.yml** — build/test using only conda-forge channel (Linux/Windows, Python 3.10-3.14) - **build-with-clang.yml** — Linux Clang compiler compatibility validation +- **build-with-standard-clang.yml** — standard Clang compiler compatibility validation +- **build_pip.yml** — validates editable build - **pre-commit.yml** — code quality checks (flake8, etc.) - **openssf-scorecard.yml** — security posture scanning diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b87b389..53df9cf 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,7 +37,7 @@ Higher-precedence file overrides; lower must not restate overridden guidance. - Build/config: `pyproject.toml`, `setup.py` - Recipe/deps: `conda-recipe/meta.yaml`, `conda-recipe/conda_build_config.yaml` - CI: `.github/workflows/*.{yml,yaml}` -- API contracts: `mkl/__init__.py`, `mkl/_mkl_service.pyx` +- API contracts: `mkl/__init__.py`, `mkl/_py_mkl_service.pyx` - Tests: `mkl/tests/test_mkl_service.py` ## MKL-specific constraints diff --git a/AGENTS.md b/AGENTS.md index 5d422ef..f4d15bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,21 +15,24 @@ Originally part of Intel® Distribution for Python*, now a standalone package av ## Key components - **Python interface:** `mkl/__init__.py` — public API surface -- **Cython wrapper:** `mkl/_mkl_service.pyx` — wraps MKL support functions +- **Cython wrapper:** `mkl/_py_mkl_service.pyx` — wraps MKL support functions - **C init module:** `mkl/_mklinitmodule.c` — Linux-side MKL runtime preloading / initialization - **Helper:** `mkl/_init_helper.py` — Windows venv DLL loading helper -- **Build system:** setuptools + Cython +- **Build system:** meson-python + Cython ## Build dependencies **Required:** - Intel® oneMKL +- meson-python +- CMake +- Ninja - Cython - Python 3.10+ **Conda environment:** ```bash -conda install -c conda-forge mkl-devel cython -python setup.py install +conda install -c conda-forge mkl-devel cython meson-python cmake ninja +python -m pip install --no-deps --no-build-isolation . ``` ## CI/CD @@ -37,7 +40,10 @@ python setup.py install - **Python versions:** 3.10, 3.11, 3.12, 3.13, 3.14 - **Workflows:** `.github/workflows/` - `conda-package.yml` — main conda build/test pipeline + - `conda-package-cf.yml — conda build/test using only conda-forge channel - `build-with-clang.yml` — Linux Clang compatibility + - `build-with-standard-clang.yml` — standard Clang compiler compatibility validation + - `build_pip` — validates editable build - `pre-commit.yml` — code quality checks - `openssf-scorecard.yml` — security scanning @@ -85,4 +91,6 @@ Below directories have local `AGENTS.md` for deeper context: For broader IntelPython ecosystem context, see: - `mkl_umath` (MKL-backed NumPy ufuncs) - `mkl_random` (MKL-based random number generation) +- `mkl_fft` (MKL-based fast fourier transform functions) - `dpnp` (Data Parallel NumPy) +- `dpctl` (Data Parallel Control) diff --git a/mkl/AGENTS.md b/mkl/AGENTS.md index 9710b83..f72405d 100644 --- a/mkl/AGENTS.md +++ b/mkl/AGENTS.md @@ -4,8 +4,8 @@ Core Python/Cython implementation: MKL support function wrappers and runtime con ## Structure - `__init__.py` — public API, RTLD_GLOBAL context manager, module initialization -- `_mkl_service.pyx` — Cython wrappers for MKL support functions -- `_mkl_service.pxd` — Cython declarations (C function signatures) +- `_py_mkl_service.pyx` — Cython wrappers for MKL support functions +- `_py_mkl_service.pxd` — Cython declarations (C function signatures) - `_mklinitmodule.c` — C extension for Linux-side MKL runtime preloading/init - `_init_helper.py` — Windows loading helper (DLL path setup in venv) - `_version.py` — version string (dynamic via setuptools) @@ -41,7 +41,7 @@ Core Python/Cython implementation: MKL support function wrappers and runtime con - **RTLD_GLOBAL preload path:** Linux preload is handled in `_mklinitmodule.c`; Windows DLL setup is in `_init_helper.py` ## Cython details -- `_mkl_service.pyx` → generates `_py_mkl_service` extension module +- `_py_mkl_service.pyx` → generates `_py_mkl_service` extension module - `.pxd` file declares external C functions from MKL headers - Cython build requires MKL headers (`mkl-devel`) From 2d34258354acc771655d0513d26269b53e8cb6aa Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 19:56:57 -0700 Subject: [PATCH 14/21] add MKLMemory object, backed with mkl_malloc memory exposes Python buffer protocol --- meson.build | 13 +++++- mkl/__init__.py | 1 + mkl/_mkl_memory.pyx | 104 +++++++++++++++++++++++++++++++++++++++++++ mkl/_mkl_service.pxd | 2 + 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 mkl/_mkl_memory.pyx diff --git a/meson.build b/meson.build index 1ea4a7f..90db4a7 100644 --- a/meson.build +++ b/meson.build @@ -43,7 +43,7 @@ py.extension_module( subdir: 'mkl' ) -# Cython extension +# Cython extensions py.extension_module( '_py_mkl_service', sources: ['mkl/_py_mkl_service.pyx'], @@ -54,6 +54,17 @@ py.extension_module( subdir: 'mkl' ) +py.extension_module( + '_mkl_memory', + sources: ['mkl/_mkl_memory.pyx'], + dependencies: [mkl_dep], + c_args: c_args, + install_rpath: rpath, + install: true, + subdir: 'mkl' +) + + # Python sources py.install_sources( [ diff --git a/mkl/__init__.py b/mkl/__init__.py index c0eb2ae..beadbfc 100644 --- a/mkl/__init__.py +++ b/mkl/__init__.py @@ -57,6 +57,7 @@ def __exit__(self, *args): del RTLD_for_MKL +from ._mkl_memory import MKLMemory from ._py_mkl_service import ( cbwr_get, cbwr_get_auto_branch, diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx new file mode 100644 index 0000000..bc4a87d --- /dev/null +++ b/mkl/_mkl_memory.pyx @@ -0,0 +1,104 @@ +# Copyright (c) 2018, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# distutils: language = c +# cython: language_level=3 + +import numbers + +from cpython cimport Py_buffer +from libc.string cimport memcpy + +from mkl._mkl_service cimport mkl_malloc, mkl_free + + +cdef class MKLMemory: + cdef void *_memory_ptr + cdef Py_ssize_t nbytes + + cdef _cinit_empty(self): + self._memory_ptr = NULL + self.nbytes = 0 + + cdef _cinit_alloc(self, Py_ssize_t nbytes, Py_ssize_t alignment): + self._cinit_empty() + + if (nbytes > 0): + with nogil: + p = mkl_malloc(nbytes, alignment) + + if (p): + self._memory_ptr = p + self.nbytes = nbytes + else: + raise MemoryError( + "MKL memory allocation failed." + ) + else: + raise ValueError( + "Number of bytes of request allocation must be positive." + ) + + cdef _cinit_other(self, object other, Py_ssize_t alignment): + cdef MKLMemory other_mem + if isinstance(other, MKLMemory): + other_mem = other + else: + raise ValueError( + f"Argument {other} is not of type MKLMemory." + ) + self._cinit_alloc(other_mem.nbytes, alignment) + with nogil: + memcpy(self._memory_ptr, other_mem._memory_ptr, self.nbytes) + + def __cinit__(self, other, *, Py_ssize_t alignment=64): + if isinstance(other, numbers.Integral): + self._cinit_alloc(other, alignment) + else: + self._cinit_other(other, alignment) + + def __dealloc__(self): + if not (self._memory_ptr is NULL): + mkl_free(self._memory_ptr) + self._cinit_empty() + + cdef void *get_data_ptr(self): + return self._memory_ptr + + def __getbuffer__(self, Py_buffer *buffer, int flags): + buffer.buf = self._memory_ptr + buffer.format = "B" # byte + buffer.internal = NULL # see References + buffer.itemsize = 1 + buffer.len = self.nbytes + buffer.ndim = 1 + buffer.obj = self + buffer.readonly = 0 + buffer.shape = &self.nbytes + buffer.strides = &buffer.itemsize + buffer.suboffsets = NULL # for pointer arrays only + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/mkl/_mkl_service.pxd b/mkl/_mkl_service.pxd index 4cba157..a96afe6 100644 --- a/mkl/_mkl_service.pxd +++ b/mkl/_mkl_service.pxd @@ -148,6 +148,8 @@ cdef extern from "mkl.h": MKL_INT64 mkl_mem_stat(int* buf) MKL_INT64 mkl_peak_mem_usage(int mode) int mkl_set_memory_limit(int mem_type, size_t limit) + void *mkl_malloc(size_t size, int alignment) nogil + void mkl_free(void *ptr) nogil # Conditional Numerical Reproducibility int mkl_cbwr_set(int settings) From 4eba4f39cdd6c49e464d9553d236c07587c9e6d0 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 21:10:25 -0700 Subject: [PATCH 15/21] add realloc to MKLMemory as atomics are a c11+ feature, specific flags are needed to enable on window --- meson.build | 8 ++++++++ mkl/_mkl_memory.pyx | 31 +++++++++++++++++++++++++++++-- mkl/_mkl_service.pxd | 1 + 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index 90db4a7..638a0c3 100644 --- a/meson.build +++ b/meson.build @@ -7,6 +7,7 @@ project( check: true ).stdout().strip(), default_options: [ + 'c_std=c11', 'buildtype=release', ] ) @@ -18,6 +19,13 @@ c_args = ['-DNDEBUG'] thread_dep = dependency('threads') cc = meson.get_compiler('c') +if cc.get_id() == 'msvc' + add_project_arguments( + '/experimental:c11atomics', + language: 'c' + ) +endif + mkl_dep = dependency('MKL', method: 'cmake', modules: ['MKL::MKL'], cmake_args: [ diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx index bc4a87d..c6450d4 100644 --- a/mkl/_mkl_memory.pyx +++ b/mkl/_mkl_memory.pyx @@ -31,16 +31,25 @@ import numbers from cpython cimport Py_buffer from libc.string cimport memcpy -from mkl._mkl_service cimport mkl_malloc, mkl_free +from mkl._mkl_service cimport mkl_malloc, mkl_realloc, mkl_free + +cdef extern from "stdatomic.h" nogil: + ctypedef int atomic_int "_Atomic int" + void atomic_init(atomic_int *obj, int value) + int atomic_fetch_add(atomic_int *obj, int value) + int atomic_fetch_sub(atomic_int *obj, int value) + int atomic_load(atomic_int *obj) cdef class MKLMemory: cdef void *_memory_ptr cdef Py_ssize_t nbytes + cdef atomic_int exported_buffers cdef _cinit_empty(self): self._memory_ptr = NULL self.nbytes = 0 + atomic_init(&self.exported_buffers, 0) cdef _cinit_alloc(self, Py_ssize_t nbytes, Py_ssize_t alignment): self._cinit_empty() @@ -100,5 +109,23 @@ cdef class MKLMemory: buffer.strides = &buffer.itemsize buffer.suboffsets = NULL # for pointer arrays only + atomic_fetch_add(&self.exported_buffers, 1) + def __releasebuffer__(self, Py_buffer *buffer): - pass + atomic_fetch_sub(&self.exported_buffers, 1) + + def realloc(self, Py_ssize_t new_nbytes): + if atomic_load(&self.exported_buffers) > 0: + raise BufferError("Cannot realloc memory while there are exported buffers.") + if new_nbytes <= 0: + raise ValueError("New number of bytes must be positive.") + + cdef void *p + with nogil: + p = mkl_realloc(self._memory_ptr, new_nbytes) + + if not p: + raise MemoryError("MKL memory reallocation failed.") + + self._memory_ptr = p + self.nbytes = new_nbytes diff --git a/mkl/_mkl_service.pxd b/mkl/_mkl_service.pxd index a96afe6..fc3c070 100644 --- a/mkl/_mkl_service.pxd +++ b/mkl/_mkl_service.pxd @@ -149,6 +149,7 @@ cdef extern from "mkl.h": MKL_INT64 mkl_peak_mem_usage(int mode) int mkl_set_memory_limit(int mem_type, size_t limit) void *mkl_malloc(size_t size, int alignment) nogil + void *mkl_realloc(void *ptr, size_t size) nogil void mkl_free(void *ptr) nogil # Conditional Numerical Reproducibility From 3a678f1661a9c4a91b3d65829f5ad9dcb0d08e76 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 21:18:53 -0700 Subject: [PATCH 16/21] overload MKLMemory constructor to use mkl_calloc --- mkl/_mkl_memory.pyx | 74 +++++++++++++++++++++++++++++++++++--------- mkl/_mkl_service.pxd | 1 + 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx index c6450d4..49a2365 100644 --- a/mkl/_mkl_memory.pyx +++ b/mkl/_mkl_memory.pyx @@ -31,7 +31,8 @@ import numbers from cpython cimport Py_buffer from libc.string cimport memcpy -from mkl._mkl_service cimport mkl_malloc, mkl_realloc, mkl_free +from mkl._mkl_service cimport mkl_calloc, mkl_free, mkl_malloc, mkl_realloc + cdef extern from "stdatomic.h" nogil: ctypedef int atomic_int "_Atomic int" @@ -51,7 +52,7 @@ cdef class MKLMemory: self.nbytes = 0 atomic_init(&self.exported_buffers, 0) - cdef _cinit_alloc(self, Py_ssize_t nbytes, Py_ssize_t alignment): + cdef _cinit_malloc(self, Py_ssize_t nbytes, Py_ssize_t alignment): self._cinit_empty() if (nbytes > 0): @@ -67,26 +68,71 @@ cdef class MKLMemory: ) else: raise ValueError( - "Number of bytes of request allocation must be positive." + "Number of bytes of requested allocation must be positive." ) - cdef _cinit_other(self, object other, Py_ssize_t alignment): - cdef MKLMemory other_mem - if isinstance(other, MKLMemory): - other_mem = other + cdef _cinit_calloc(self, Py_ssize_t num, Py_ssize_t size, Py_ssize_t alignment): + self._cinit_empty() + + if (num > 0 and size > 0): + with nogil: + p = mkl_calloc(num, size, alignment) + + if (p): + self._memory_ptr = p + self.nbytes = num * size + else: + raise MemoryError( + "MKL memory allocation failed." + ) else: raise ValueError( - f"Argument {other} is not of type MKLMemory." + "Number of elements and size of requested allocation must be " + "positive." ) - self._cinit_alloc(other_mem.nbytes, alignment) + + cdef _cinit_mklmemory(self, object other, Py_ssize_t alignment): + other_mem = other + + self._cinit_malloc(other_mem.nbytes, alignment) with nogil: memcpy(self._memory_ptr, other_mem._memory_ptr, self.nbytes) - def __cinit__(self, other, *, Py_ssize_t alignment=64): - if isinstance(other, numbers.Integral): - self._cinit_alloc(other, alignment) - else: - self._cinit_other(other, alignment) + def __cinit__(self, *args, **kwargs): + cdef Py_ssize_t alignment = kwargs.get("alignment", 64) + + n_args = len(args) + if not (0 < n_args < 3): + raise TypeError( + "MKLMemory constructor takes 1 or 2 arguments, but " + f"{n_args} were given" + ) + if n_args == 1: + arg = args[0] + if isinstance(arg, numbers.Integral): + self._cinit_malloc(arg, alignment) + elif isinstance(arg, MKLMemory): + self._cinit_mklmemory(arg, alignment) + else: + raise TypeError( + "MKLMemory single argument constructor expects an integer " + f"or MKLMemory instance, but got {type(arg)}" + ) + + elif n_args == 2: + arg0, arg1 = args[0], args[1] + if not isinstance(arg0, numbers.Integral): + raise TypeError( + "MKLMemory constructor expects first argument " + f"to be an integer, but got {type(arg0)}" + ) + if not isinstance(arg1, numbers.Integral): + raise TypeError( + "MKLMemory constructor expects second argument " + f"to be an integer, but got {type(arg1)}" + ) + + self._cinit_calloc(arg0, arg1, alignment) def __dealloc__(self): if not (self._memory_ptr is NULL): diff --git a/mkl/_mkl_service.pxd b/mkl/_mkl_service.pxd index fc3c070..8edde28 100644 --- a/mkl/_mkl_service.pxd +++ b/mkl/_mkl_service.pxd @@ -150,6 +150,7 @@ cdef extern from "mkl.h": int mkl_set_memory_limit(int mem_type, size_t limit) void *mkl_malloc(size_t size, int alignment) nogil void *mkl_realloc(void *ptr, size_t size) nogil + void *mkl_calloc(size_t num, size_t size, int alignment) nogil void mkl_free(void *ptr) nogil # Conditional Numerical Reproducibility From e224ae931ad6700e924ffebc475572e20fc4c87a Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 22:10:59 -0700 Subject: [PATCH 17/21] add info properties to MKLMemory --- mkl/_mkl_memory.pyx | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx index 49a2365..d63bf8b 100644 --- a/mkl/_mkl_memory.pyx +++ b/mkl/_mkl_memory.pyx @@ -175,3 +175,27 @@ cdef class MKLMemory: self._memory_ptr = p self.nbytes = new_nbytes + + @property + def nbytes(self): + return self.nbytes + + @property + def size(self): + return self.nbytes + + @property + def _pointer(self): + return (self._memory_ptr) + + def __repr__(self): + return ( + f"(self._memory_ptr))}>" + ) + + def __len__(self): + return self.nbytes + + def __sizeof__(self): + return self.nbytes From 9082613a513b42e6c2c7bd308b91a0e50695df78 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 23:39:23 -0700 Subject: [PATCH 18/21] add pickling support for MKLMemory --- mkl/_mkl_memory.pyx | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx index d63bf8b..12bb1e3 100644 --- a/mkl/_mkl_memory.pyx +++ b/mkl/_mkl_memory.pyx @@ -42,14 +42,29 @@ cdef extern from "stdatomic.h" nogil: int atomic_load(atomic_int *obj) +def _mkl_memory_from_bytes(bytes data, Py_ssize_t alignment): + cdef Py_ssize_t nbytes = len(data) + cdef MKLMemory mem = MKLMemory(nbytes, alignment=alignment) + + cdef void *dst = mem._memory_ptr + cdef char *src = data + + with nogil: + memcpy(dst, src, nbytes) + + return mem + + cdef class MKLMemory: cdef void *_memory_ptr cdef Py_ssize_t nbytes + cdef Py_ssize_t alignment cdef atomic_int exported_buffers cdef _cinit_empty(self): self._memory_ptr = NULL self.nbytes = 0 + self.alignment = 0 atomic_init(&self.exported_buffers, 0) cdef _cinit_malloc(self, Py_ssize_t nbytes, Py_ssize_t alignment): @@ -62,6 +77,7 @@ cdef class MKLMemory: if (p): self._memory_ptr = p self.nbytes = nbytes + self.alignment = alignment else: raise MemoryError( "MKL memory allocation failed." @@ -81,6 +97,7 @@ cdef class MKLMemory: if (p): self._memory_ptr = p self.nbytes = num * size + self.alignment = alignment else: raise MemoryError( "MKL memory allocation failed." @@ -176,6 +193,10 @@ cdef class MKLMemory: self._memory_ptr = p self.nbytes = new_nbytes + def tobytes(self): + cdef char* data_ptr = self._memory_ptr + return data_ptr[:self.nbytes] + @property def nbytes(self): return self.nbytes @@ -184,6 +205,10 @@ cdef class MKLMemory: def size(self): return self.nbytes + @property + def alignment(self): + return self.alignment + @property def _pointer(self): return (self._memory_ptr) @@ -199,3 +224,6 @@ cdef class MKLMemory: def __sizeof__(self): return self.nbytes + + def __reduce__(self): + return (_mkl_memory_from_bytes, (self.tobytes(), self.alignment)) From 549cf011765c0e69e5941ac794ed0b0ef6cabb38 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sat, 11 Apr 2026 23:53:17 -0700 Subject: [PATCH 19/21] propagate alignment in MKLMemory --- mkl/_mkl_memory.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mkl/_mkl_memory.pyx b/mkl/_mkl_memory.pyx index 12bb1e3..77b6d78 100644 --- a/mkl/_mkl_memory.pyx +++ b/mkl/_mkl_memory.pyx @@ -116,7 +116,7 @@ cdef class MKLMemory: memcpy(self._memory_ptr, other_mem._memory_ptr, self.nbytes) def __cinit__(self, *args, **kwargs): - cdef Py_ssize_t alignment = kwargs.get("alignment", 64) + cdef Py_ssize_t alignment n_args = len(args) if not (0 < n_args < 3): @@ -127,8 +127,10 @@ cdef class MKLMemory: if n_args == 1: arg = args[0] if isinstance(arg, numbers.Integral): + alignment = kwargs.get("alignment", 64) self._cinit_malloc(arg, alignment) elif isinstance(arg, MKLMemory): + alignment = kwargs.get("alignment", arg.alignment) self._cinit_mklmemory(arg, alignment) else: raise TypeError( @@ -138,6 +140,7 @@ cdef class MKLMemory: elif n_args == 2: arg0, arg1 = args[0], args[1] + alignment = kwargs.get("alignment", 64) if not isinstance(arg0, numbers.Integral): raise TypeError( "MKLMemory constructor expects first argument " @@ -148,7 +151,6 @@ cdef class MKLMemory: "MKLMemory constructor expects second argument " f"to be an integer, but got {type(arg1)}" ) - self._cinit_calloc(arg0, arg1, alignment) def __dealloc__(self): From 6567ce2be424c9ab42358757cfe680da3a4f25b8 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 12 Apr 2026 00:19:16 -0700 Subject: [PATCH 20/21] add tests for MKLMemory class --- mkl/tests/test_mkl_memory.py | 139 +++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 mkl/tests/test_mkl_memory.py diff --git a/mkl/tests/test_mkl_memory.py b/mkl/tests/test_mkl_memory.py new file mode 100644 index 0000000..96f6f6f --- /dev/null +++ b/mkl/tests/test_mkl_memory.py @@ -0,0 +1,139 @@ +# Copyright (c) 2018, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys + +import mkl + + +def test_mkl_memory_create_malloc(): + nbytes = 1024 + mem = mkl.MKLMemory(nbytes) + assert mem.nbytes == nbytes + # default alignment is 64 bytes + assert mem.alignment == 64 + + +def test_mkl_memory_create_calloc(): + size = 32 + num = 32 + nbytes = num * size + # test creating with mkl_calloc + mem = mkl.MKLMemory(num, size) + assert mem.nbytes == nbytes + # default alignment is 64 bytes + assert mem.alignment == 64 + + +def test_mkl_memory_create_with_malloc_and_alignment(): + size = 32 + num = 32 + nbytes = num * size + alignment = 128 + mem = mkl.MKLMemory(nbytes, alignment=alignment) + assert mem.nbytes == nbytes + assert mem.alignment == alignment + + +def test_mkl_memory_create_with_calloc_and_alignment(): + size = 32 + num = 32 + nbytes = num * size + alignment = 128 + mem = mkl.MKLMemory(num, size, alignment=alignment) + assert mem.nbytes == nbytes + + +def test_mkl_memory_create_from_mkl_memory(): + mem1 = mkl.MKLMemory(1024) + mem2 = mkl.MKLMemory(mem1) + assert mem2.nbytes == mem1.nbytes + + +def test_mkl_memory_create_from_mkl_memory_with_alignment(): + mem1 = mkl.MKLMemory(1024) + alignment = 128 + mem2 = mkl.MKLMemory(mem1, alignment=alignment) + assert mem2.nbytes == mem1.nbytes + assert mem2.alignment == alignment + + +def test_mkl_memory_propagates_alignment(): + mem1 = mkl.MKLMemory(1024, alignment=128) + mem2 = mkl.MKLMemory(mem1) + assert mem2.nbytes == mem1.nbytes + assert mem2.alignment == mem1.alignment + + +def test_mkl_memory_properties(): + nbytes = 1024 + mem = mkl.MKLMemory(nbytes) + assert len(mem) == nbytes + assert type(repr(mem)) is str + assert type(bytes(mem)) is bytes + assert sys.getsizeof(mem) >= nbytes + + +def test_buffer_protocol(): + mem = mkl.MKLMemory(1024) + mv1 = memoryview(mem) + assert mv1.nbytes == mem.nbytes + mv2 = memoryview(mem) + assert mv1 == mv2 + + +def test_pickling(): + import pickle + + mem = mkl.MKLMemory(1024) + mv = memoryview(mem) + for i in range(len(mem)): + mv[i] = (i % 32) + ord("a") + + mem_reconstructed = pickle.loads(pickle.dumps(mem)) + assert type(mem) is type(mem_reconstructed), "Pickling should preserve type" + assert ( + mem.tobytes() == mem_reconstructed.tobytes() + ), "Pickling should preserve buffer content" + assert ( + mem._pointer != mem_reconstructed._pointer + ), "Pickling/unpickling should be changing pointer" + + +def test_pickling_with_alignment(): + import pickle + + mem = mkl.MKLMemory(1024, alignment=128) + mem_reconstructed = pickle.loads(pickle.dumps(mem)) + assert type(mem) is type(mem_reconstructed), "Pickling should preserve type" + assert ( + mem.tobytes() == mem_reconstructed.tobytes() + ), "Pickling should preserve buffer content" + assert ( + mem._pointer != mem_reconstructed._pointer + ), "Pickling/unpickling should be changing pointer" + assert ( + mem.alignment == mem_reconstructed.alignment + ), "Pickling should preserve alignment" From 00fea26618f668e4a2553f8735b5b61a5dbfd974 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Sun, 12 Apr 2026 03:07:07 -0700 Subject: [PATCH 21/21] add nogil to MKL functions for freeing buffers --- mkl/_mkl_service.pxd | 10 +++++----- mkl/_py_mkl_service.pyx | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mkl/_mkl_service.pxd b/mkl/_mkl_service.pxd index 8edde28..3e07630 100644 --- a/mkl/_mkl_service.pxd +++ b/mkl/_mkl_service.pxd @@ -29,7 +29,7 @@ ctypedef unsigned long long MKL_UINT64 ctypedef int MKL_INT -cdef extern from "mkl.h": +cdef extern from "mkl.h" nogil: # MKL Function Domains Constants int MKL_DOMAIN_BLAS int MKL_DOMAIN_FFT @@ -148,10 +148,10 @@ cdef extern from "mkl.h": MKL_INT64 mkl_mem_stat(int* buf) MKL_INT64 mkl_peak_mem_usage(int mode) int mkl_set_memory_limit(int mem_type, size_t limit) - void *mkl_malloc(size_t size, int alignment) nogil - void *mkl_realloc(void *ptr, size_t size) nogil - void *mkl_calloc(size_t num, size_t size, int alignment) nogil - void mkl_free(void *ptr) nogil + void *mkl_malloc(size_t size, int alignment) + void *mkl_realloc(void *ptr, size_t size) + void *mkl_calloc(size_t num, size_t size, int alignment) + void mkl_free(void *ptr) # Conditional Numerical Reproducibility int mkl_cbwr_set(int settings) diff --git a/mkl/_py_mkl_service.pyx b/mkl/_py_mkl_service.pyx index 975a454..69c2f52 100644 --- a/mkl/_py_mkl_service.pyx +++ b/mkl/_py_mkl_service.pyx @@ -601,7 +601,8 @@ cdef inline void __free_buffers() noexcept: """ Frees unused memory allocated by the Intel(R) MKL Memory Allocator. """ - mkl.mkl_free_buffers() + with nogil: + mkl.mkl_free_buffers() return @@ -610,7 +611,8 @@ cdef inline void __thread_free_buffers() noexcept: Frees unused memory allocated by the Intel(R) MKL Memory Allocator in the current thread. """ - mkl.mkl_thread_free_buffers() + with nogil: + mkl.mkl_thread_free_buffers() return