From 39fbe2be69396beb81fa89e962b70f6c001a2926 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Tue, 12 Dec 2023 17:19:47 +0200 Subject: [PATCH 1/7] Change to pyproject.toml file --- src/pyclblast/pyproject.toml | 38 ++++++++++++++++++++++++++++++++++++ src/pyclblast/setup.py | 19 +----------------- 2 files changed, 39 insertions(+), 18 deletions(-) create mode 100644 src/pyclblast/pyproject.toml diff --git a/src/pyclblast/pyproject.toml b/src/pyclblast/pyproject.toml new file mode 100644 index 00000000..42a5e582 --- /dev/null +++ b/src/pyclblast/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "cython", + "numpy", + "cmake>=3.20" +] +build-backend = "setuptools.build_meta" + +[project] +name = "pyclblast" +version = "1.4.0" +description = "Python bindings for CLBlast, the tuned OpenCL BLAS library" +authors = [ + {name = "Cedric Nugteren", email = "web@cedricnugteren.nl"} +] +license = {text = "Apache Software License"} +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", +] +keywords = ["OpenCL", "BLAS", "CLBlast", "GEMM", "matrix-multiplication"] +dependencies = [ + "numpy", + "pyopencl" +] + +[project.urls] +Homepage = "https://github.com/CNugteren/CLBlast/blob/master/src/pyclblast" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/src/pyclblast/setup.py b/src/pyclblast/setup.py index c6811935..35042b50 100644 --- a/src/pyclblast/setup.py +++ b/src/pyclblast/setup.py @@ -38,24 +38,7 @@ setup( name="pyclblast", - version="1.4.0", - author="Cedric Nugteren", - author_email="web@cedricnugteren.nl", - url="https://github.com/CNugteren/CLBlast/blob/master/src/pyclblast", - description="Python bindings for CLBlast, the tuned OpenCL BLAS library", - license="Apache Software License", - requires=["numpy", "pyopencl", "cython"], - package_dir={'': 'src'}, scripts=[], ext_modules=ext_modules, - cmdclass={"build_ext": build_ext}, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - ], - keywords="OpenCL BLAS CLBlast GEMM matrix-multiplication" + cmdclass={"build_ext": build_ext} ) From 5dc9feaf3aac9a30b807e6c725477c5838a211e6 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Tue, 12 Dec 2023 17:26:07 +0200 Subject: [PATCH 2/7] Switch to cmake for building the extension --- src/pyclblast/CMakeLists.txt | 61 +++++++++++++++++++++ src/pyclblast/MANIFEST.in | 1 + src/pyclblast/setup.py | 102 +++++++++++++++++++++++++---------- 3 files changed, 135 insertions(+), 29 deletions(-) create mode 100644 src/pyclblast/CMakeLists.txt diff --git a/src/pyclblast/CMakeLists.txt b/src/pyclblast/CMakeLists.txt new file mode 100644 index 00000000..c421fbef --- /dev/null +++ b/src/pyclblast/CMakeLists.txt @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.20) +project(pyclblast) + +# Find python and numpy +find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) + +# Run the cython compiler +cmake_path(APPEND CMAKE_CURRENT_SOURCE_DIR "./src" + OUTPUT_VARIABLE Cython_SOURCE_DIR) +execute_process(COMMAND ${Python3_EXECUTABLE} + "-m" "cython" "-3" "-o" "pyclblast.cpp" "pyclblast.pyx" + WORKING_DIRECTORY "${Cython_SOURCE_DIR}" + RESULT_VARIABLE Cython_RUN) +if (Cython_RUN) + message("Cython run returned ${Cython_RUN}") + message(FATAL_ERROR "Cython compiler failed") +endif() + + +# Add module target +Python3_add_library(pyclblast MODULE WITH_SOABI ./src/pyclblast.cpp) + +# Numpy libraries - NOTE: clean NPY_LIBRARIES cache (may fail for venv) +cmake_path(GET Python3_NumPy_INCLUDE_DIRS PARENT_PATH Python3_NumPy_CORE_DIR) +unset(NPY_LIBRARIES CACHE) +find_library(NPY_LIBRARIES + NAMES npymath + PATHS ${Python3_NumPy_CORE_DIR} + PATH_SUFFIXES lib + DOC "Numpy math library" + REQUIRED + NO_DEFAULT_PATH) +target_link_libraries(pyclblast PRIVATE ${NPY_LIBRARIES}) +target_include_directories(pyclblast PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) + +# CLBlast library +# TODO: Add hints for CLBlast package discovery +find_package(CLBlast CONFIG REQUIRED) +target_link_libraries(pyclblast PRIVATE clblast) + +# In windows pyclblast cannot find the dll, even on path. +# Probably related to change in 3.8, that loads dll only for trusted location +# see https://stackoverflow.com/questions/41365446/how-to-resolve-importerror-dll-load-failed-on-python +# One workaround is to copy the dll to the same dir as the module. +# TODO: add python version check +if (WIN32) + cmake_path(APPEND CLBlast_DIR "../../../bin" OUTPUT_VARIABLE CLBlast_BINDIR) + cmake_path(SET CLBlast_BINDIR NORMALIZE "${CLBlast_BINDIR}") + unset(CLBlast_SHARED_LIBPATH CACHE) + find_file(CLBlast_SHARED_LIBPATH + NAMES clblast.dll + PATHS ${CLBlast_BINDIR} + DOC "CLBlast shared library" + REQUIRED) + + # copy dll to build + add_custom_command(TARGET pyclblast POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CLBlast_SHARED_LIBPATH} + ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}) +endif() diff --git a/src/pyclblast/MANIFEST.in b/src/pyclblast/MANIFEST.in index fb20923f..853ceff1 100644 --- a/src/pyclblast/MANIFEST.in +++ b/src/pyclblast/MANIFEST.in @@ -1,2 +1,3 @@ include README.md setup.py src/*.pyx include samples/*.py +include CMakeLists.txt diff --git a/src/pyclblast/setup.py b/src/pyclblast/setup.py index 35042b50..2be8ed53 100644 --- a/src/pyclblast/setup.py +++ b/src/pyclblast/setup.py @@ -5,40 +5,84 @@ # Author(s): # Cedric Nugteren -from setuptools import setup - -from distutils.extension import Extension -from Cython.Distutils import build_ext +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext import platform -import numpy import os +import subprocess +import sys + + +# Command line flags forwarded to CMake +cmake_cmd_args = [] +for arg in sys.argv: + if arg.startswith('-D'): + cmake_cmd_args.append(arg) + +for arg in cmake_cmd_args: + sys.argv.remove(arg) + + +# For setup.py with Cmake extensions, see +# https://martinopilia.com/posts/2018/09/15/building-python-extension.html +class CMakeExtension(Extension): + def __init__(self, name, cmake_lists_dir='.', **kwargs): + Extension.__init__(self, name, sources=[], **kwargs) + self.cmake_lists_dir = os.path.abspath(cmake_lists_dir) + + +class CMakeBuildExt(build_ext): + def build_extensions(self): + # Ensure that CMake is present and working + try: + subprocess.check_output(['cmake', '--version']) + except OSError: + raise RuntimeError('Cannot find CMake executable') + + for ext in self.extensions: + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE={}'.format(extdir), + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE={}'.format(self.build_temp) + ] + + # We can handle some platform-specific settings at our discretion + if platform.system() == 'Windows': + plat = ('x64' if platform.architecture()[0] == '64bit' else 'Win32') + cmake_args += [ + # These options are likely to be needed under Windows + '-DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=TRUE', + '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE={}'.format(extdir), + ] + # Assuming that Visual Studio and MinGW are supported compilers + if self.compiler.compiler_type == 'msvc': + cmake_args += [ + '-DCMAKE_GENERATOR_PLATFORM=%s' % plat, + ] + else: + cmake_args += [ + '-G', 'MinGW Makefiles', + ] + + cmake_args += cmake_cmd_args + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + # Config + subprocess.check_call(['cmake', ext.cmake_lists_dir] + cmake_args, + cwd=self.build_temp) + + # Build + subprocess.check_call(['cmake', '--build', '.', '--config', 'Release'], + cwd=self.build_temp) -np_incdir = numpy.get_include() -np_libdir = os.path.join(np_incdir, '..', 'lib', '') - -runtime_library_dirs = list() -if platform.system() == "Linux": - runtime_library_dirs.append("/usr/local/lib") -elif platform.system() == "Windows": - runtime_library_dirs.append("C:/Program Files/clblast/lib") - runtime_library_dirs.append("C:/Program Files (x86)/clblast/lib") - -ext_modules = list() -ext_modules.append( - Extension( - "pyclblast", - ["src/pyclblast.pyx"], - libraries=["clblast", "npymath"], - runtime_library_dirs=runtime_library_dirs, - library_dirs=[np_libdir], - include_dirs=[np_incdir], - language="c++" - ) -) setup( name="pyclblast", scripts=[], - ext_modules=ext_modules, - cmdclass={"build_ext": build_ext} + ext_modules=[CMakeExtension("pyclblast")], + cmdclass={"build_ext": CMakeBuildExt} ) From 218058651eaf44848ee569c715c497ef23a2cf67 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Tue, 12 Dec 2023 17:33:50 +0200 Subject: [PATCH 3/7] Update readme --- src/pyclblast/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyclblast/README.md b/src/pyclblast/README.md index fe4cd7e3..23932826 100644 --- a/src/pyclblast/README.md +++ b/src/pyclblast/README.md @@ -53,6 +53,6 @@ How to release a new version on PyPi Following [the guide](https://packaging.python.org/tutorials/packaging-projects/), in essence doing (after changing the version number in `setup.py`): - python3 setup.py sdist bdist_wheel + python3 -m build python3 -m twine upload --repository pypi dist/pyclblast-1.4.0.tar.gz # use '__token__' as username and supply the token from your PyPi account From a17925c4bd2440ab09f82597411c3310874424eb Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Fri, 29 Dec 2023 19:16:11 +0200 Subject: [PATCH 4/7] Update CHANGELOG --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index cc919b77..d4155a4b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,8 @@ Development version (next version) * Convert float scalar values to cl_half for fp16 routines * Amax/amin, max/min routines accept unsigned integer buffers for index - Generator script now always use LF endings, independent of the platform +- Switch to pyproject.toml file for installing python bindings +- Build python binding using Cmake, adding Windows support Version 1.6.1 - Fix pointer error in pyclblast on Arm From 116313cfe60a5679ec6ec0fa360e900418fd1d10 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Fri, 19 Jan 2024 17:43:57 +0200 Subject: [PATCH 5/7] Add hint fot CLBlast discovery --- src/pyclblast/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pyclblast/CMakeLists.txt b/src/pyclblast/CMakeLists.txt index c421fbef..05b89354 100644 --- a/src/pyclblast/CMakeLists.txt +++ b/src/pyclblast/CMakeLists.txt @@ -34,8 +34,11 @@ target_link_libraries(pyclblast PRIVATE ${NPY_LIBRARIES}) target_include_directories(pyclblast PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) # CLBlast library -# TODO: Add hints for CLBlast package discovery -find_package(CLBlast CONFIG REQUIRED) +set(CLBLAST_HINTS + ${CLBLAST_ROOT} + $ENV{CLBLAST_ROOT} +) +find_package(CLBlast CONFIG REQUIRED HINTS ${CLBLAST_HINTS}) target_link_libraries(pyclblast PRIVATE clblast) # In windows pyclblast cannot find the dll, even on path. From 7e40840277f6d23d598f1bdc66f4a2074985f974 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Fri, 19 Jan 2024 17:50:58 +0200 Subject: [PATCH 6/7] Update README about detecting the library --- src/pyclblast/README.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/pyclblast/README.md b/src/pyclblast/README.md index 23932826..c150d30f 100644 --- a/src/pyclblast/README.md +++ b/src/pyclblast/README.md @@ -15,29 +15,39 @@ Non-Python requirements: * OpenCL * [CLBlast](https://github.com/CNugteren/CLBlast) -Python requirements: - -* Cython -* [PyOpenCL](https://github.com/pyopencl/pyopencl/) - Getting started ------------- -After installation OpenCL and CLBlast, simply use pip to install PyCLBlast, e.g.: +After installing OpenCL and CLBlast, simply use pip to install PyCLBlast, e.g.: pip install --user pyclblast -To start using the library, browse the [CLBlast](https://github.com/CNugteren/CLBlast) documentation or check out the PyCLBlast samples provides in the `samples` subfolder. +To start using the library, browse the [CLBlast](https://github.com/CNugteren/CLBlast) documentation or check out the PyCLBlast samples provided in the `samples` subfolder. + +For developers, install CLBlast and [cython](https://cython.org/) (e.g. in a Python3 virtualenv): + + pip install Cython + +And then compile the bindings from this location using pip: + + pip install . + + +Detecting CLBlast +------------- + +The CLBlast library should be present and detectable to your system, to successfully install the PyCLBlast bindings. In some systems, this is done automatically. But if the CLBlast library cannot be detected, the PyCLBlast installation will fail. To ensure detection, one can apply either of the following: -For developers, first install CLBlast, followed by the Python requirements (e.g. in a Python3 virtualenv): +* Add the CLBLast root directory to the environment path. +* Create the environment variable `CLBLAST_ROOT` that holds the path to the CLBLast root directory. +* Define the `cmake` variables `CMAKE_PREFIX_PATH` or the `CLBLAST_ROOT` variable that point to the CLBlast root directory, as: - pip install Cython numpy pybind11 - pip install pyopencl + pip install . -C skbuild.cmake.args="-DCMAKE_PREFIX_PATH=/root/path/to/clblast" -And then compile the library from this location using the `setup.py` file: +* Create the environment variable `CLBlast_DIR` that holds the path to the directory where either of the `CLBlastConfig.cmake` or `clblast-config.cmake` files reside. - python setup.py install +Note that the aforementioned environment variables should be set only during the installation of PyCLBlast and can be unset during normal use. Testing PyCLBlast From b00e107fc36bf92d0c04fbf7524fbee31be1f382 Mon Sep 17 00:00:00 2001 From: Thomas Vasileiou Date: Fri, 19 Jan 2024 18:18:42 +0200 Subject: [PATCH 7/7] Switch to scikit-build-core for using CMake --- src/pyclblast/CMakeLists.txt | 33 ++++++++------ src/pyclblast/MANIFEST.in | 2 +- src/pyclblast/pyproject.toml | 10 +--- src/pyclblast/setup.py | 88 ------------------------------------ 4 files changed, 21 insertions(+), 112 deletions(-) delete mode 100644 src/pyclblast/setup.py diff --git a/src/pyclblast/CMakeLists.txt b/src/pyclblast/CMakeLists.txt index 05b89354..0fa67297 100644 --- a/src/pyclblast/CMakeLists.txt +++ b/src/pyclblast/CMakeLists.txt @@ -1,24 +1,28 @@ cmake_minimum_required(VERSION 3.20) -project(pyclblast) +project(${SKBUILD_PROJECT_NAME} LANGUAGES CXX) # Find python and numpy -find_package(Python3 REQUIRED COMPONENTS Interpreter Development NumPy) +find_package( + Python3 + REQUIRED + COMPONENTS Interpreter Development.Module NumPy +) # Run the cython compiler cmake_path(APPEND CMAKE_CURRENT_SOURCE_DIR "./src" OUTPUT_VARIABLE Cython_SOURCE_DIR) -execute_process(COMMAND ${Python3_EXECUTABLE} - "-m" "cython" "-3" "-o" "pyclblast.cpp" "pyclblast.pyx" - WORKING_DIRECTORY "${Cython_SOURCE_DIR}" - RESULT_VARIABLE Cython_RUN) -if (Cython_RUN) - message("Cython run returned ${Cython_RUN}") - message(FATAL_ERROR "Cython compiler failed") -endif() +find_program(CYTHON "cython") +add_custom_command( + OUTPUT "${Cython_SOURCE_DIR}/pyclblast.cpp" + DEPENDS "${Cython_SOURCE_DIR}/pyclblast.pyx" + VERBATIM + COMMAND "${CYTHON}" -3 "${Cython_SOURCE_DIR}/pyclblast.pyx" + --output-file "${Cython_SOURCE_DIR}/pyclblast.cpp") # Add module target -Python3_add_library(pyclblast MODULE WITH_SOABI ./src/pyclblast.cpp) +Python3_add_library(pyclblast MODULE WITH_SOABI + "${Cython_SOURCE_DIR}/pyclblast.cpp") # Numpy libraries - NOTE: clean NPY_LIBRARIES cache (may fail for venv) cmake_path(GET Python3_NumPy_INCLUDE_DIRS PARENT_PATH Python3_NumPy_CORE_DIR) @@ -41,6 +45,8 @@ set(CLBLAST_HINTS find_package(CLBlast CONFIG REQUIRED HINTS ${CLBLAST_HINTS}) target_link_libraries(pyclblast PRIVATE clblast) +install(TARGETS pyclblast DESTINATION .) + # In windows pyclblast cannot find the dll, even on path. # Probably related to change in 3.8, that loads dll only for trusted location # see https://stackoverflow.com/questions/41365446/how-to-resolve-importerror-dll-load-failed-on-python @@ -57,8 +63,5 @@ if (WIN32) REQUIRED) # copy dll to build - add_custom_command(TARGET pyclblast POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CLBlast_SHARED_LIBPATH} - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE}) + install(FILES ${CLBlast_SHARED_LIBPATH} DESTINATION .) endif() diff --git a/src/pyclblast/MANIFEST.in b/src/pyclblast/MANIFEST.in index 853ceff1..bd198ad2 100644 --- a/src/pyclblast/MANIFEST.in +++ b/src/pyclblast/MANIFEST.in @@ -1,3 +1,3 @@ -include README.md setup.py src/*.pyx +include README.md src/*.pyx include samples/*.py include CMakeLists.txt diff --git a/src/pyclblast/pyproject.toml b/src/pyclblast/pyproject.toml index 42a5e582..02b1aac5 100644 --- a/src/pyclblast/pyproject.toml +++ b/src/pyclblast/pyproject.toml @@ -1,12 +1,6 @@ [build-system] -requires = [ - "setuptools>=42", - "wheel", - "cython", - "numpy", - "cmake>=3.20" -] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core", "cython", "numpy"] +build-backend = "scikit_build_core.build" [project] name = "pyclblast" diff --git a/src/pyclblast/setup.py b/src/pyclblast/setup.py deleted file mode 100644 index 2be8ed53..00000000 --- a/src/pyclblast/setup.py +++ /dev/null @@ -1,88 +0,0 @@ - -# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. -# This file follows the PEP8 Python style guide and uses a max-width of 100 characters per line. -# -# Author(s): -# Cedric Nugteren - -from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext -import platform -import os -import subprocess -import sys - - -# Command line flags forwarded to CMake -cmake_cmd_args = [] -for arg in sys.argv: - if arg.startswith('-D'): - cmake_cmd_args.append(arg) - -for arg in cmake_cmd_args: - sys.argv.remove(arg) - - -# For setup.py with Cmake extensions, see -# https://martinopilia.com/posts/2018/09/15/building-python-extension.html -class CMakeExtension(Extension): - def __init__(self, name, cmake_lists_dir='.', **kwargs): - Extension.__init__(self, name, sources=[], **kwargs) - self.cmake_lists_dir = os.path.abspath(cmake_lists_dir) - - -class CMakeBuildExt(build_ext): - def build_extensions(self): - # Ensure that CMake is present and working - try: - subprocess.check_output(['cmake', '--version']) - except OSError: - raise RuntimeError('Cannot find CMake executable') - - for ext in self.extensions: - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - - cmake_args = [ - '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE={}'.format(extdir), - '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE={}'.format(self.build_temp) - ] - - # We can handle some platform-specific settings at our discretion - if platform.system() == 'Windows': - plat = ('x64' if platform.architecture()[0] == '64bit' else 'Win32') - cmake_args += [ - # These options are likely to be needed under Windows - '-DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=TRUE', - '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE={}'.format(extdir), - ] - # Assuming that Visual Studio and MinGW are supported compilers - if self.compiler.compiler_type == 'msvc': - cmake_args += [ - '-DCMAKE_GENERATOR_PLATFORM=%s' % plat, - ] - else: - cmake_args += [ - '-G', 'MinGW Makefiles', - ] - - cmake_args += cmake_cmd_args - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - - # Config - subprocess.check_call(['cmake', ext.cmake_lists_dir] + cmake_args, - cwd=self.build_temp) - - # Build - subprocess.check_call(['cmake', '--build', '.', '--config', 'Release'], - cwd=self.build_temp) - - -setup( - name="pyclblast", - scripts=[], - ext_modules=[CMakeExtension("pyclblast")], - cmdclass={"build_ext": CMakeBuildExt} -)