From 78fb0d9c612e551218bf727cd7c8fff0f848e6b9 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Fri, 29 Sep 2023 14:03:51 +0100 Subject: [PATCH 01/10] Add spatialindex headerfiles to package --- MANIFEST.in | 1 + ci/install_libspatialindex.bash | 26 ++++++++++++++++++++------ ci/install_libspatialindex.bat | 2 ++ pyproject.toml | 2 +- setup.py | 2 +- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 16653343..5a2ac335 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include MANIFEST.in recursive-include benchmarks * recursive-include tests * recursive-include docs * +recursive-include rtree/include *.h diff --git a/ci/install_libspatialindex.bash b/ci/install_libspatialindex.bash index 0174f33e..4b38503e 100755 --- a/ci/install_libspatialindex.bash +++ b/ci/install_libspatialindex.bash @@ -7,7 +7,7 @@ SHA256=63a03bfb26aa65cf0159f925f6c3491b6ef79bc0e3db5a631d96772d6541187e # where to copy resulting files # this has to be run before `cd`-ing anywhere -gentarget() { +libtarget() { OURPWD=$PWD cd "$(dirname "$0")" mkdir -p ../rtree/lib @@ -17,6 +17,16 @@ gentarget() { echo $arr } +headertarget() { + OURPWD=$PWD + cd "$(dirname "$0")" + mkdir -p ../rtree/include + cd ../rtree/include + arr=$(pwd) + cd "$OURPWD" + echo $arr +} + scriptloc() { OURPWD=$PWD cd "$(dirname "$0")" @@ -26,7 +36,8 @@ scriptloc() { } # note that we're doing this convoluted thing to get # an absolute path so mac doesn't yell at us -TARGET=`gentarget` +LIBTARGET=`libtarget` +HEADERTARGET=`headertarget` SL=`scriptloc` rm $VERSION.zip || true @@ -60,10 +71,13 @@ if [ "$(uname)" == "Darwin" ]; then # change the rpath in the dylib to point to the same directory install_name_tool -change @rpath/libspatialindex.6.dylib @loader_path/libspatialindex.dylib bin/libspatialindex_c.dylib # copy the dylib files to the target director - cp bin/libspatialindex.dylib $TARGET - cp bin/libspatialindex_c.dylib $TARGET + cp bin/libspatialindex.dylib $LIBTARGET + cp bin/libspatialindex_c.dylib $LIBTARGET + cp -r ../include/* $HEADERTARGET else - cp -d bin/* $TARGET + cp -L bin/* $LIBTARGET + cp -r ../include/* $HEADERTARGET fi -ls $TARGET +ls $LIBTARGET +ls -R $HEADERTARGET diff --git a/ci/install_libspatialindex.bat b/ci/install_libspatialindex.bat index c08d3677..c91f9d5b 100755 --- a/ci/install_libspatialindex.bat +++ b/ci/install_libspatialindex.bat @@ -21,7 +21,9 @@ ninja mkdir %~dp0\..\rtree\lib copy bin\*.dll %~dp0\..\rtree\lib +copy ..\/include/* %~dp0\..\rtree\include rmdir /Q /S bin dir %~dp0\..\rtree\ dir %~dp0\..\rtree\lib +dir %~dp0\..\rtree\include diff --git a/pyproject.toml b/pyproject.toml index aac4283a..29bfbf50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ zip-safe = false version = {attr = "rtree.__version__"} [tool.setuptools.package-data] -rtree = ["lib", "py.typed"] +rtree = ["lib", "include", "py.typed"] [tool.black] target-version = ["py38", "py39", "py310", "py311", "py312"] diff --git a/setup.py b/setup.py index 98887ddc..13a3df44 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def finalize_options(self) -> None: source_dir = os.path.join(_cwd, "rtree", "lib") # what patterns represent shared libraries - patterns = {"*.so", "libspatialindex*dylib", "*.dll"} + patterns = {"*.so*", "libspatialindex*dylib", "*.dll"} if not os.path.isdir(source_dir): # no copying of binary parts to library From 5ec23e7ef95920937e477f0b5c2caf9fea16b165 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Thu, 5 Oct 2023 12:10:43 +0100 Subject: [PATCH 02/10] Add library and header introspection --- rtree/core.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rtree/core.py b/rtree/core.py index 1bd2b80d..7f619e96 100644 --- a/rtree/core.py +++ b/rtree/core.py @@ -1,6 +1,7 @@ from __future__ import annotations import ctypes +import os from . import finder from .exceptions import RTreeError @@ -72,9 +73,23 @@ def free_error_msg_ptr(result, func, cargs): return retvalue +def get_include(): + import rtree + return os.path.join(os.path.dirname(rtree.__file__), "include") + + # load the shared library by looking in likely places rt = finder.load() + +def get_libraries(): + return os.path.dirname(rt._name) + + +def get_library_name(): + return os.path.split(rt._name)[-1] + + rt.SIDX_Version.argtypes = [] rt.SIDX_Version.restype = ctypes.POINTER(ctypes.c_char) rt.SIDX_Version.errcheck = free_returned_char_p # type: ignore From 8a9556416be1381c7739cf1e374980e716bd5c12 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Wed, 8 Nov 2023 19:00:48 +0000 Subject: [PATCH 03/10] Fix CI fails --- ci/install_libspatialindex.bat | 2 +- rtree/core.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/install_libspatialindex.bat b/ci/install_libspatialindex.bat index c91f9d5b..0bfa4581 100755 --- a/ci/install_libspatialindex.bat +++ b/ci/install_libspatialindex.bat @@ -21,7 +21,7 @@ ninja mkdir %~dp0\..\rtree\lib copy bin\*.dll %~dp0\..\rtree\lib -copy ..\/include/* %~dp0\..\rtree\include +copy ..\include\* %~dp0\..\rtree\include rmdir /Q /S bin dir %~dp0\..\rtree\ diff --git a/rtree/core.py b/rtree/core.py index 7f619e96..700abd15 100644 --- a/rtree/core.py +++ b/rtree/core.py @@ -75,6 +75,7 @@ def free_error_msg_ptr(result, func, cargs): def get_include(): import rtree + return os.path.join(os.path.dirname(rtree.__file__), "include") From ab8a38659b3a6374b83fab217b5333b42c34113d Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Wed, 8 Nov 2023 23:43:19 +0000 Subject: [PATCH 04/10] Downloaded Windows VM to test this one --- ci/install_libspatialindex.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install_libspatialindex.bat b/ci/install_libspatialindex.bat index 0bfa4581..a2ce9a95 100755 --- a/ci/install_libspatialindex.bat +++ b/ci/install_libspatialindex.bat @@ -21,7 +21,7 @@ ninja mkdir %~dp0\..\rtree\lib copy bin\*.dll %~dp0\..\rtree\lib -copy ..\include\* %~dp0\..\rtree\include +xcopy /S ..\include\* %~dp0\..\rtree\include\ rmdir /Q /S bin dir %~dp0\..\rtree\ From 002c3a456e60ca4be1f4fa6be54dae90b79d2ed1 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Fri, 24 Nov 2023 18:33:35 +0000 Subject: [PATCH 05/10] Allow get_include to handle sdist installations --- rtree/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rtree/core.py b/rtree/core.py index 700abd15..27e870ca 100644 --- a/rtree/core.py +++ b/rtree/core.py @@ -1,4 +1,5 @@ from __future__ import annotations +from importlib import resources import ctypes import os @@ -76,7 +77,8 @@ def free_error_msg_ptr(result, func, cargs): def get_include(): import rtree - return os.path.join(os.path.dirname(rtree.__file__), "include") + include_dir = resources.files(rtree).joinpath("include") + return str(include_dir) if include_dir.is_dir() else "" # load the shared library by looking in likely places From 5e390b36502a98533be780e1ea15b661dddb8394 Mon Sep 17 00:00:00 2001 From: Mike Taves Date: Fri, 29 Dec 2023 20:39:26 +1300 Subject: [PATCH 06/10] Avoid packaging header data for sdist --- MANIFEST.in | 1 - pyproject.toml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5a2ac335..16653343 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,3 @@ include MANIFEST.in recursive-include benchmarks * recursive-include tests * recursive-include docs * -recursive-include rtree/include *.h diff --git a/pyproject.toml b/pyproject.toml index 29bfbf50..84873643 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,12 +39,13 @@ Repository = "https://github.com/Toblerity/rtree" [tool.setuptools] packages = ["rtree"] zip-safe = false +include-package-data = false [tool.setuptools.dynamic] version = {attr = "rtree.__version__"} [tool.setuptools.package-data] -rtree = ["lib", "include", "py.typed"] +rtree = ["py.typed"] [tool.black] target-version = ["py38", "py39", "py310", "py311", "py312"] From 82d8278b79ecb099cd9b0596737790327691b178 Mon Sep 17 00:00:00 2001 From: Mike Taves Date: Fri, 29 Dec 2023 21:48:09 +1300 Subject: [PATCH 07/10] Enable install.finalize_options to copy lib and include to wheel --- setup.py | 99 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index 13a3df44..db474146 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -import os +import sys +from pathlib import Path from setuptools import setup from setuptools.command.install import install @@ -7,7 +8,7 @@ from wheel.bdist_wheel import bdist_wheel as _bdist_wheel # current working directory of this setup.py file -_cwd = os.path.abspath(os.path.split(__file__)[0]) +_cwd = Path(__file__).resolve().parent class bdist_wheel(_bdist_wheel): # type: ignore[misc] @@ -26,54 +27,60 @@ def has_ext_modules(foo) -> bool: class InstallPlatlib(install): # type: ignore[misc] def finalize_options(self) -> None: """ - Copy the shared libraries into the wheel. Note that this - will *only* check in `rtree/lib` rather than anywhere on - the system so if you are building a wheel you *must* copy or - symlink the `.so`/`.dll`/`.dylib` files into `rtree/lib`. + Copy the shared libraries and header files into the wheel. Note that + this will *only* check in `rtree/lib` and `include` rather than + anywhere on the system so if you are building a wheel you *must* copy + or symlink the `.so`/`.dll`/`.dylib` files into `rtree/lib` and + `.h` into `rtree/include`. """ - # use for checking extension types - from fnmatch import fnmatch - install.finalize_options(self) if self.distribution.has_ext_modules(): self.install_lib = self.install_platlib - # now copy over libspatialindex - # get the location of the shared library on the filesystem - - # where we're putting the shared library in the build directory - target_dir = os.path.join(self.build_lib, "rtree", "lib") - # where are we checking for shared libraries - source_dir = os.path.join(_cwd, "rtree", "lib") - - # what patterns represent shared libraries - patterns = {"*.so*", "libspatialindex*dylib", "*.dll"} - - if not os.path.isdir(source_dir): - # no copying of binary parts to library - # this is so `pip install .` works even - # if `rtree/lib` isn't populated - return - - for file_name in os.listdir(source_dir): - # make sure file name is lower case - check = file_name.lower() - # use filename pattern matching to see if it is - # a shared library format file - if not any(fnmatch(check, p) for p in patterns): - continue - - # if the source isn't a file skip it - if not os.path.isfile(os.path.join(source_dir, file_name)): - continue - - # make build directory if it doesn't exist yet - if not os.path.isdir(target_dir): - os.makedirs(target_dir) - - # copy the source file to the target directory - self.copy_file( - os.path.join(source_dir, file_name), os.path.join(target_dir, file_name) - ) + + # source files to copy + source_dir = _cwd / "rtree" + + # destination for the files in the build directory + target_dir = Path(self.build_lib) / "rtree" + + source_lib = source_dir / "lib" + target_lib = target_dir / "lib" + if source_lib.is_dir(): + # what patterns represent shared libraries for supported platforms + if sys.platform.startswith("win"): + lib_pattern = "*.dll" + elif sys.platform.startswith("linux"): + lib_pattern = "*.so*" + elif sys.platform == "darwin": + lib_pattern = "libspatialindex*dylib" + else: + raise ValueError(f"unhandled platform {sys.platform!r}") + + for pth in source_lib.glob(lib_pattern): + # if the source isn't a file skip it + if not pth.is_file(): + continue + + # make directory if it doesn't exist yet + if not target_lib.is_dir(): + target_lib.mkdir(parents=True) + + # copy the source file to the target directory + self.copy_file(str(pth), str(target_lib / pth.name)) + + source_include = source_dir / "include" + target_include = target_dir / "include" + if source_include.is_dir(): + for pth in source_include.rglob("*.h"): + rpth = pth.relative_to(source_include) + target_subdir = target_include / rpth + + # make directory if it doesn't exist yet + if not target_subdir.is_dir(): + target_subdir.mkdir(parents=True) + + # copy the source file to the target directory + self.copy_file(str(pth), str(target_include / rpth)) # See pyproject.toml for other project metadata From 1ccc2eb7dc06524c4f5c8040eb9c5219218c2a32 Mon Sep 17 00:00:00 2001 From: Mike Taves Date: Wed, 17 Jan 2024 13:36:24 +1300 Subject: [PATCH 08/10] Remove rtree.core.* functions --- rtree/core.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/rtree/core.py b/rtree/core.py index 27e870ca..1bd2b80d 100644 --- a/rtree/core.py +++ b/rtree/core.py @@ -1,8 +1,6 @@ from __future__ import annotations -from importlib import resources import ctypes -import os from . import finder from .exceptions import RTreeError @@ -74,25 +72,9 @@ def free_error_msg_ptr(result, func, cargs): return retvalue -def get_include(): - import rtree - - include_dir = resources.files(rtree).joinpath("include") - return str(include_dir) if include_dir.is_dir() else "" - - # load the shared library by looking in likely places rt = finder.load() - -def get_libraries(): - return os.path.dirname(rt._name) - - -def get_library_name(): - return os.path.split(rt._name)[-1] - - rt.SIDX_Version.argtypes = [] rt.SIDX_Version.restype = ctypes.POINTER(ctypes.c_char) rt.SIDX_Version.errcheck = free_returned_char_p # type: ignore From 855d8e07d0700250b202530bd0901d1d253dc141 Mon Sep 17 00:00:00 2001 From: Mike Taves Date: Wed, 17 Jan 2024 14:21:51 +1300 Subject: [PATCH 09/10] Rewrite parts of finder to use Pathlib; add get_include() and tests --- rtree/finder.py | 105 +++++++++++++++++++++++++++++-------------- tests/test_finder.py | 19 ++++++++ 2 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 tests/test_finder.py diff --git a/rtree/finder.py b/rtree/finder.py index 10899bca..f852e813 100644 --- a/rtree/finder.py +++ b/rtree/finder.py @@ -7,22 +7,22 @@ from __future__ import annotations import ctypes +import importlib.metadata import os import platform import sys from ctypes.util import find_library +from pathlib import Path -# the current working directory of this file -_cwd = os.path.abspath(os.path.expanduser(os.path.dirname(__file__))) +_cwd = Path(__file__).parent +_sys_prefix = Path(sys.prefix) # generate a bunch of candidate locations where the # libspatialindex shared library *might* be hanging out -_candidates = [ - os.environ.get("SPATIALINDEX_C_LIBRARY", None), - os.path.join(_cwd, "lib"), - _cwd, - "", -] +_candidates = [] +if "SPATIALINDEX_C_LIBRARY" in os.environ: + _candidates.append(Path(os.environ["SPATIALINDEX_C_LIBRARY"])) +_candidates += [_cwd / "lib", _cwd, Path("")] def load() -> ctypes.CDLL: @@ -39,29 +39,26 @@ def load() -> ctypes.CDLL: lib_name = f"spatialindex_c-{arch}.dll" # add search paths for conda installs - if ( - os.path.exists(os.path.join(sys.prefix, "conda-meta")) - or "conda" in sys.version - ): - _candidates.append(os.path.join(sys.prefix, "Library", "bin")) + if (_sys_prefix / "conda-meta").exists() or "conda" in sys.version: + _candidates.append(_sys_prefix / "Library" / "bin") # get the current PATH oldenv = os.environ.get("PATH", "").strip().rstrip(";") # run through our list of candidate locations for path in _candidates: - if not path or not os.path.exists(path): + if not path.exists(): continue # temporarily add the path to the PATH environment variable # so Windows can find additional DLL dependencies. - os.environ["PATH"] = ";".join([path, oldenv]) + os.environ["PATH"] = ";".join([str(path), oldenv]) try: - rt = ctypes.cdll.LoadLibrary(os.path.join(path, lib_name)) + rt = ctypes.cdll.LoadLibrary(str(path / lib_name)) if rt is not None: return rt except OSError: pass - except BaseException as E: - print(f"rtree.finder unexpected error: {E!s}") + except BaseException as err: + print(f"rtree.finder unexpected error: {err!s}", file=sys.stderr) finally: os.environ["PATH"] = oldenv raise OSError(f"could not find or load {lib_name}") @@ -73,8 +70,6 @@ def load() -> ctypes.CDLL: # macos shared libraries are `.dylib` lib_name = "libspatialindex_c.dylib" else: - import importlib.metadata - # linux shared libraries are `.so` lib_name = "libspatialindex_c.so" @@ -88,7 +83,7 @@ def load() -> ctypes.CDLL: and file.stem.startswith("libspatialindex") and ".so" in file.suffixes ): - _candidates.insert(1, os.path.join(str(file.locate()))) + _candidates.insert(1, Path(file.locate())) break except importlib.metadata.PackageNotFoundError: pass @@ -96,41 +91,83 @@ def load() -> ctypes.CDLL: # get the starting working directory cwd = os.getcwd() for cand in _candidates: - if cand is None: - continue - elif os.path.isdir(cand): + if cand.is_dir(): # if our candidate is a directory use best guess path = cand - target = os.path.join(cand, lib_name) - elif os.path.isfile(cand): + target = cand / lib_name + elif cand.is_file(): # if candidate is just a file use that - path = os.path.split(cand)[0] + path = cand.parent target = cand else: continue - if not os.path.exists(target): + if not target.exists(): continue try: # move to the location we're checking os.chdir(path) # try loading the target file candidate - rt = ctypes.cdll.LoadLibrary(target) + rt = ctypes.cdll.LoadLibrary(str(target)) if rt is not None: return rt - except BaseException as E: - print(f"rtree.finder ({target}) unexpected error: {E!s}") + except BaseException as err: + print( + f"rtree.finder ({target}) unexpected error: {err!s}", + file=sys.stderr, + ) finally: os.chdir(cwd) try: # try loading library using LD path search - path = find_library("spatialindex_c") - if path is not None: - return ctypes.cdll.LoadLibrary(path) + pth = find_library("spatialindex_c") + if pth is not None: + return ctypes.cdll.LoadLibrary(pth) except BaseException: pass raise OSError("Could not load libspatialindex_c library") + + +def get_include() -> str: + """Return the directory that contains the spatialindex \\*.h files. + + :returns: Path to include directory or "" if not found. + """ + # check if was bundled with a binary wheel + try: + pkg_files = importlib.metadata.files("rtree") + if pkg_files is not None: + for path in pkg_files: # type: ignore + if path.name == "SpatialIndex.h": + return str(path.parent.parent) + except importlib.metadata.PackageNotFoundError: + pass + + # look for this header file in a few directories + path_to_spatialindex_h = Path("include/spatialindex/SpatialIndex.h") + + # check sys.prefix, e.g. conda's libspatialindex package + if os.name == "nt": + file = _sys_prefix / "Library" / path_to_spatialindex_h + else: + file = _sys_prefix / path_to_spatialindex_h + if file.is_file(): + return str(file.parent.parent) + + # check if relative to lib + libdir = Path(load()._name).parent + file = libdir.parent / path_to_spatialindex_h + if file.is_file(): + return str(file.parent.parent) + + # check system install + file = Path("/usr") / path_to_spatialindex_h + if file.is_file(): + return str(file.parent.parent) + + # not found + return "" diff --git a/tests/test_finder.py b/tests/test_finder.py new file mode 100644 index 00000000..4a05ad2f --- /dev/null +++ b/tests/test_finder.py @@ -0,0 +1,19 @@ +from ctypes import CDLL +from pathlib import Path + +from rtree import finder + + +def test_load(): + lib = finder.load() + assert isinstance(lib, CDLL) + + +def test_get_include(): + incl = finder.get_include() + assert isinstance(incl, str) + if incl: + path = Path(incl) + assert path.is_dir() + assert (path / "spatialindex").is_dir() + assert (path / "spatialindex" / "SpatialIndex.h").is_file() From ed7ecaec59392de9059f814fb2224b197c2a2791 Mon Sep 17 00:00:00 2001 From: Mike Taves Date: Wed, 17 Jan 2024 17:01:57 +1300 Subject: [PATCH 10/10] Fix absolute path from wheel; fix copy header bug --- rtree/finder.py | 2 +- setup.py | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/rtree/finder.py b/rtree/finder.py index f852e813..e03d2127 100644 --- a/rtree/finder.py +++ b/rtree/finder.py @@ -143,7 +143,7 @@ def get_include() -> str: if pkg_files is not None: for path in pkg_files: # type: ignore if path.name == "SpatialIndex.h": - return str(path.parent.parent) + return str(Path(path.locate()).parent.parent) except importlib.metadata.PackageNotFoundError: pass diff --git a/setup.py b/setup.py index db474146..a1fc40d9 100755 --- a/setup.py +++ b/setup.py @@ -56,15 +56,12 @@ def finalize_options(self) -> None: else: raise ValueError(f"unhandled platform {sys.platform!r}") + target_lib.mkdir(parents=True, exist_ok=True) for pth in source_lib.glob(lib_pattern): # if the source isn't a file skip it if not pth.is_file(): continue - # make directory if it doesn't exist yet - if not target_lib.is_dir(): - target_lib.mkdir(parents=True) - # copy the source file to the target directory self.copy_file(str(pth), str(target_lib / pth.name)) @@ -73,14 +70,11 @@ def finalize_options(self) -> None: if source_include.is_dir(): for pth in source_include.rglob("*.h"): rpth = pth.relative_to(source_include) - target_subdir = target_include / rpth - - # make directory if it doesn't exist yet - if not target_subdir.is_dir(): - target_subdir.mkdir(parents=True) # copy the source file to the target directory - self.copy_file(str(pth), str(target_include / rpth)) + target_subdir = target_include / rpth.parent + target_subdir.mkdir(parents=True, exist_ok=True) + self.copy_file(str(pth), str(target_subdir)) # See pyproject.toml for other project metadata