From 87a32be7feedc284d00acb303cad7da2d7a14a03 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Fri, 10 May 2024 00:58:37 +0530 Subject: [PATCH] recipe: introduce `PyProjectRecipe` and `MesonRecipe` (to update `pandas`, `numpy` and other recipes) (#3007) --- Dockerfile | 2 + Makefile | 2 +- doc/source/quickstart.rst | 2 + pythonforandroid/recipe.py | 302 ++++++++++++------ pythonforandroid/recipes/Pillow/__init__.py | 2 +- pythonforandroid/recipes/cppy/__init__.py | 14 - .../recipes/cryptography/__init__.py | 8 +- .../recipes/kiwisolver/__init__.py | 9 +- .../recipes/matplotlib/__init__.py | 15 +- .../recipes/matplotlib/skip_macos.patch | 12 + pythonforandroid/recipes/numpy/__init__.py | 94 ++---- .../add_libm_explicitly_to_build.patch | 20 -- .../recipes/numpy/patches/ranlib.patch | 11 - .../numpy/patches/remove-default-paths.patch | 28 -- pythonforandroid/recipes/pandas/__init__.py | 28 +- .../recipes/pandas/fix_numpy_includes.patch | 111 +++++-- .../recipes/pydantic-core/__init__.py | 2 - pythonforandroid/recipes/scipy/__init__.py | 1 + pythonforandroid/recommendations.py | 2 +- setup.py | 2 +- testapps/on_device_unit_tests/setup.py | 6 +- 21 files changed, 376 insertions(+), 297 deletions(-) delete mode 100644 pythonforandroid/recipes/cppy/__init__.py create mode 100644 pythonforandroid/recipes/matplotlib/skip_macos.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/add_libm_explicitly_to_build.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/ranlib.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/remove-default-paths.patch diff --git a/Dockerfile b/Dockerfile index b5b2c597ff..408a0802f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN ${RETRY} apt -y update -qq > /dev/null \ ant \ autoconf \ automake \ + autopoint \ ccache \ cmake \ g++ \ @@ -70,6 +71,7 @@ RUN ${RETRY} apt -y update -qq > /dev/null \ make \ openjdk-17-jdk \ patch \ + patchelf \ pkg-config \ python3 \ python3-dev \ diff --git a/Makefile b/Makefile index 19a2379ab2..6e30ee5936 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ testapps-with-scipy/%: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY) && \ python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --requirements python3,scipy,kivy \ + --requirements python3,scipy,kivy \ --arch=armeabi-v7a --arch=arm64-v8a testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 987a00cddb..81c860f888 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -72,6 +72,7 @@ the following command (re-adapted from the `Dockerfile` we use to perform CI bui ant \ autoconf \ automake \ + autopoint \ ccache \ cmake \ g++ \ @@ -85,6 +86,7 @@ the following command (re-adapted from the `Dockerfile` we use to perform CI bui make \ openjdk-17-jdk \ patch \ + patchelf \ pkg-config \ python3 \ python3-dev \ diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index d70571ad08..c4131c53e6 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,4 +1,4 @@ -from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split, sep +from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split import glob import hashlib @@ -12,6 +12,8 @@ from urllib.request import urlretrieve from os import listdir, unlink, environ, curdir, walk from sys import stdout +from wheel.wheelfile import WheelFile +from wheel.cli.tags import tags as wheel_tags import time try: from urlparse import urlparse @@ -232,7 +234,7 @@ def report_hook(index, blksize, size): shprint(sh.git, 'clone', '--recursive', url, target) with current_directory(target): if self.version: - shprint(sh.git, 'fetch', '--depth', '1', 'origin', self.version) + shprint(sh.git, 'fetch', '--tags', '--depth', '1') shprint(sh.git, 'checkout', self.version) branch = sh.git('branch', '--show-current') if branch: @@ -478,10 +480,11 @@ def unpack(self, arch): elif isdir(extraction_filename): ensure_dir(directory_name) for entry in listdir(extraction_filename): - if entry not in ('.git',): - shprint(sh.cp, '-Rv', - join(extraction_filename, entry), - directory_name) + # Previously we filtered out the .git folder, but during the build process for some recipes + # (e.g. when version is parsed by `setuptools_scm`) that may be needed. + shprint(sh.cp, '-Rv', + join(extraction_filename, entry), + directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}' @@ -843,7 +846,6 @@ class PythonRecipe(Recipe): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if 'python3' not in self.depends: # We ensure here that the recipe depends on python even it overrode # `depends`. We only do this if it doesn't already depend on any @@ -893,12 +895,12 @@ def folder_name(self): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) - env['PYTHONNOUSERSITE'] = '1' - # Set the LANG, this isn't usually important but is a better default # as it occasionally matters how Python e.g. reads files env['LANG'] = "en_GB.UTF-8" + # Binaries made by packages installed by pip + env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"] if not self.call_hostpython_via_targetpython: env['CFLAGS'] += ' -I{}'.format( @@ -979,26 +981,37 @@ def install_hostpython_package(self, arch): _env=env, *self.setup_extra_args) @property - def python_version(self): - return Recipe.get_recipe("python3", self.ctx).version + def python_major_minor_version(self): + parsed_version = packaging.version.parse(self.ctx.python_recipe.version) + return f"{parsed_version.major}.{parsed_version.minor}" + + def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): + if not packages: + packages = self.hostpython_prerequisites - def install_hostpython_prerequisites(self, force_upgrade=True): - if len(self.hostpython_prerequisites) == 0: + if len(packages) == 0: return + pip_options = [ "install", - *self.hostpython_prerequisites, + *packages, "--target", self.hostpython_site_dir, "--python-version", - self.python_version, + self.ctx.python_recipe.version, # Don't use sources, instead wheels "--only-binary=:all:", - "--no-deps" ] if force_upgrade: pip_options.append("--upgrade") # Use system's pip shprint(sh.pip, *pip_options) + def restore_hostpython_prerequisites(self, packages): + _packages = [] + for package in packages: + original_version = Recipe.get_recipe(package, self.ctx).version + _packages.append(package + "==" + original_version) + self.install_hostpython_prerequisites(packages=_packages) + class CompiledComponentsPythonRecipe(PythonRecipe): pre_build_ext = False @@ -1010,6 +1023,7 @@ def build_arch(self, arch): calling setup.py install with the target Python dir. ''' Recipe.build_arch(self, arch) + self.install_hostpython_prerequisites() self.build_compiled_components(arch) self.install_python_package(arch) @@ -1157,7 +1171,182 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): return env -class RustCompiledComponentsRecipe(PythonRecipe): +class PyProjectRecipe(PythonRecipe): + '''Recipe for projects which containes `pyproject.toml`''' + + # Extra args to pass to `python -m build ...` + extra_build_args = [] + call_hostpython_via_targetpython = False + + def get_recipe_env(self, arch, **kwargs): + # Custom hostpython + self.ctx.python_recipe.python_exe = join( + self.ctx.python_recipe.get_build_dir(arch), "android-build", "python3") + env = super().get_recipe_env(arch, **kwargs) + build_dir = self.get_build_dir(arch) + ensure_dir(build_dir) + build_opts = join(build_dir, "build-opts.cfg") + + with open(build_opts, "w") as file: + file.write("[bdist_wheel]\nplat-name={}".format( + self.get_wheel_platform_tag(arch) + )) + file.close() + + env["DIST_EXTRA_CONFIG"] = build_opts + return env + + def get_wheel_platform_tag(self, arch): + return "android_" + { + "armeabi-v7a": "arm", + "arm64-v8a": "aarch64", + "x86_64": "x86_64", + "x86": "i686", + }[arch.arch] + + def install_wheel(self, arch, built_wheels): + _wheel = built_wheels[0] + built_wheel_dir = dirname(_wheel) + # Fix wheel platform tag + wheel_tag = wheel_tags( + _wheel, + platform_tags=self.get_wheel_platform_tag(arch), + remove=True, + ) + selected_wheel = join(built_wheel_dir, wheel_tag) + + _dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False) + if _dev_wheel_dir: + ensure_dir(_dev_wheel_dir) + shprint(sh.cp, selected_wheel, _dev_wheel_dir) + + info(f"Installing built wheel: {wheel_tag}") + destination = self.ctx.get_python_install_dir(arch.arch) + with WheelFile(selected_wheel) as wf: + for zinfo in wf.filelist: + wf.extract(zinfo, destination) + wf.close() + + def build_arch(self, arch): + self.install_hostpython_prerequisites( + packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites + ) + build_dir = self.get_build_dir(arch.arch) + env = self.get_recipe_env(arch, with_flags_in_cc=True) + # make build dir separatly + sub_build_dir = join(build_dir, "p4a_android_build") + ensure_dir(sub_build_dir) + # copy hostpython to built python to ensure correct selection of libs and includes + shprint(sh.cp, self.real_hostpython_location, self.ctx.python_recipe.python_exe) + + build_args = [ + "-m", + "build", + "--wheel", + "--config-setting", + "builddir={}".format(sub_build_dir), + ] + self.extra_build_args + + built_wheels = [] + with current_directory(build_dir): + shprint( + sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env + ) + built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")] + self.install_wheel(arch, built_wheels) + + +class MesonRecipe(PyProjectRecipe): + '''Recipe for projects which uses meson as build system''' + + meson_version = "1.4.0" + ninja_version = "1.11.1.1" + + def sanitize_flags(self, *flag_strings): + return " ".join(flag_strings).strip().split(" ") + + def get_recipe_meson_options(self, arch): + env = self.get_recipe_env(arch, with_flags_in_cc=True) + return { + "binaries": { + "c": arch.get_clang_exe(with_target=True), + "cpp": arch.get_clang_exe(with_target=True, plus_plus=True), + "ar": self.ctx.ndk.llvm_ar, + "strip": self.ctx.ndk.llvm_strip, + }, + "built-in options": { + "c_args": self.sanitize_flags(env["CFLAGS"], env["CPPFLAGS"]), + "cpp_args": self.sanitize_flags(env["CXXFLAGS"], env["CPPFLAGS"]), + "c_link_args": self.sanitize_flags(env["LDFLAGS"]), + "cpp_link_args": self.sanitize_flags(env["LDFLAGS"]), + }, + "properties": { + "needs_exe_wrapper": True, + "sys_root": self.ctx.ndk.sysroot + }, + "host_machine": { + "cpu_family": { + "arm64-v8a": "aarch64", + "armeabi-v7a": "arm", + "x86_64": "x86_64", + "x86": "x86" + }[arch.arch], + "cpu": { + "arm64-v8a": "aarch64", + "armeabi-v7a": "armv7", + "x86_64": "x86_64", + "x86": "i686" + }[arch.arch], + "endian": "little", + "system": "android", + } + } + + def write_build_options(self, arch): + """Writes python dict to meson config file""" + option_data = "" + build_options = self.get_recipe_meson_options(arch) + for key in build_options.keys(): + data_chunk = "[{}]".format(key) + for subkey in build_options[key].keys(): + value = build_options[key][subkey] + if isinstance(value, int): + value = str(value) + elif isinstance(value, str): + value = "'{}'".format(value) + elif isinstance(value, bool): + value = "true" if value else "false" + elif isinstance(value, list): + value = "['" + "', '".join(value) + "']" + data_chunk += "\n" + subkey + " = " + value + option_data += data_chunk + "\n\n" + return option_data + + def ensure_args(self, *args): + for arg in args: + if arg not in self.extra_build_args: + self.extra_build_args.append(arg) + + def build_arch(self, arch): + cross_file = join("/tmp", "android.meson.cross") + info("Writing cross file at: {}".format(cross_file)) + # write cross config file + with open(cross_file, "w") as file: + file.write(self.write_build_options(arch)) + file.close() + # set cross file + self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file)) + # ensure ninja and meson + for dep in [ + "ninja=={}".format(self.ninja_version), + "meson=={}".format(self.meson_version), + ]: + if dep not in self.hostpython_prerequisites: + self.hostpython_prerequisites.append(dep) + super().build_arch(arch) + + +class RustCompiledComponentsRecipe(PyProjectRecipe): # Rust toolchain codes # https://doc.rust-lang.org/nightly/rustc/platform-support.html RUST_ARCH_CODES = { @@ -1167,41 +1356,10 @@ class RustCompiledComponentsRecipe(PythonRecipe): "x86": "i686-linux-android", } - # Build python wheel using `maturin` instead - # of default `python -m build [...]` - use_maturin = False - - # Directory where to find built wheel - # For normal build: "dist/*.whl" - # For maturin: "target/wheels/*-linux_*.whl" - built_wheel_pattern = None - call_hostpython_via_targetpython = False - def __init__(self, *arg, **kwargs): - super().__init__(*arg, **kwargs) - self.append_deps_if_absent(["python3"]) - self.set_default_hostpython_deps() - if not self.built_wheel_pattern: - self.built_wheel_pattern = ( - "target/wheels/*-linux_*.whl" - if self.use_maturin - else "dist/*.whl" - ) - - def set_default_hostpython_deps(self): - if not self.use_maturin: - self.hostpython_prerequisites += ["build", "setuptools_rust", "wheel", "pyproject_hooks"] - else: - self.hostpython_prerequisites += ["maturin"] - - def append_deps_if_absent(self, deps): - for dep in deps: - if dep not in self.depends: - self.depends.append(dep) - - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) # Set rust build target build_target = self.RUST_ARCH_CODES[arch.arch] @@ -1220,7 +1378,7 @@ def get_recipe_env(self, arch): self.ctx.ndk_api, ), ) - realpython_dir = Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch) + realpython_dir = self.ctx.python_recipe.get_build_dir(arch.arch) env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format( self.ctx.get_libs_dir(arch.arch), join(realpython_dir, "android-build") @@ -1228,7 +1386,7 @@ def get_recipe_env(self, arch): env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join( realpython_dir, "android-build", "build", - "lib.linux-*-{}/".format(self.get_python_formatted_version()), + "lib.linux-*-{}/".format(self.python_major_minor_version), ))[0]) info_main("Ensuring rust build toolchain") @@ -1243,10 +1401,6 @@ def get_recipe_env(self, arch): ) return env - def get_python_formatted_version(self): - parsed_version = packaging.version.parse(self.python_version) - return f"{parsed_version.major}.{parsed_version.minor}" - def check_host_deps(self): if not hasattr(sh, "rustup"): error( @@ -1258,41 +1412,7 @@ def check_host_deps(self): def build_arch(self, arch): self.check_host_deps() - self.install_hostpython_prerequisites() - build_dir = self.get_build_dir(arch.arch) - env = self.get_recipe_env(arch) - built_wheel = None - - # Copy the exec with version info - hostpython_exec = join( - sep, - *self.hostpython_location.split(sep)[:-1], - "python{}".format(self.get_python_formatted_version()), - ) - shprint(sh.cp, self.hostpython_location, hostpython_exec) - - with current_directory(build_dir): - if self.use_maturin: - shprint( - sh.Command(join(self.hostpython_site_dir, "bin", "maturin")), - "build", "--interpreter", hostpython_exec, "--skip-auditwheel", - _env=env, - ) - else: - shprint( - sh.Command(hostpython_exec), - "-m", "build", "--no-isolation", "--skip-dependency-check", "--wheel", - _env=env, - ) - # Find the built wheel - built_wheel = realpath(glob.glob(self.built_wheel_pattern)[0]) - - info("Unzipping built wheel '{}'".format(basename(built_wheel))) - - # Unzip .whl file into site-packages - with zipfile.ZipFile(built_wheel, "r") as zip_ref: - zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch)) - info("Successfully installed '{}'".format(basename(built_wheel))) + super().build_arch(arch) class TargetPythonRecipe(Recipe): diff --git a/pythonforandroid/recipes/Pillow/__init__.py b/pythonforandroid/recipes/Pillow/__init__.py index f8f6929db5..cd770526ff 100644 --- a/pythonforandroid/recipes/Pillow/__init__.py +++ b/pythonforandroid/recipes/Pillow/__init__.py @@ -25,7 +25,7 @@ class PillowRecipe(CompiledComponentsPythonRecipe): version = '8.4.0' url = 'https://github.com/python-pillow/Pillow/archive/{version}.tar.gz' - site_packages_name = 'Pillow' + site_packages_name = 'PIL' depends = ['png', 'jpeg', 'freetype', 'setuptools'] opt_depends = ['libwebp'] patches = [join('patches', 'fix-setup.patch')] diff --git a/pythonforandroid/recipes/cppy/__init__.py b/pythonforandroid/recipes/cppy/__init__.py deleted file mode 100644 index f61e2c2516..0000000000 --- a/pythonforandroid/recipes/cppy/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class CppyRecipe(PythonRecipe): - site_packages_name = 'cppy' - version = '1.1.0' - url = 'https://github.com/nucleic/cppy/archive/{version}.zip' - call_hostpython_via_targetpython = False - # to be detected by the matplotlib install script - install_in_hostpython = True - depends = ['setuptools'] - - -recipe = CppyRecipe() diff --git a/pythonforandroid/recipes/cryptography/__init__.py b/pythonforandroid/recipes/cryptography/__init__.py index 8e476b3129..c6a91a13d7 100644 --- a/pythonforandroid/recipes/cryptography/__init__.py +++ b/pythonforandroid/recipes/cryptography/__init__.py @@ -7,12 +7,10 @@ class CryptographyRecipe(RustCompiledComponentsRecipe): name = 'cryptography' version = '42.0.1' url = 'https://github.com/pyca/cryptography/archive/refs/tags/{version}.tar.gz' - depends = ['openssl', 'six', 'setuptools', 'cffi'] - # recipe built cffi does not work on apple M1 - hostpython_prerequisites = ["semantic_version", "cffi"] + depends = ['openssl'] - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) openssl_build_dir = self.get_recipe('openssl', self.ctx).get_build_dir(arch.arch) build_target = self.RUST_ARCH_CODES[arch.arch].upper().replace("-", "_") openssl_include = "{}_OPENSSL_INCLUDE_DIR".format(build_target) diff --git a/pythonforandroid/recipes/kiwisolver/__init__.py b/pythonforandroid/recipes/kiwisolver/__init__.py index 587c2b9a49..c4c19ac257 100644 --- a/pythonforandroid/recipes/kiwisolver/__init__.py +++ b/pythonforandroid/recipes/kiwisolver/__init__.py @@ -1,11 +1,12 @@ -from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe +from pythonforandroid.recipe import PyProjectRecipe -class KiwiSolverRecipe(CppCompiledComponentsPythonRecipe): +class KiwiSolverRecipe(PyProjectRecipe): site_packages_name = 'kiwisolver' - version = '1.3.2' - url = 'https://github.com/nucleic/kiwi/archive/{version}.zip' + version = '1.4.5' + url = 'git+https://github.com/nucleic/kiwi' depends = ['cppy'] + need_stl_shared = True recipe = KiwiSolverRecipe() diff --git a/pythonforandroid/recipes/matplotlib/__init__.py b/pythonforandroid/recipes/matplotlib/__init__.py index f79cde3483..6fffd45596 100644 --- a/pythonforandroid/recipes/matplotlib/__init__.py +++ b/pythonforandroid/recipes/matplotlib/__init__.py @@ -1,18 +1,17 @@ -from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe +from pythonforandroid.recipe import PyProjectRecipe from pythonforandroid.util import ensure_dir from os.path import join import shutil -class MatplotlibRecipe(CppCompiledComponentsPythonRecipe): - - version = '3.5.2' +class MatplotlibRecipe(PyProjectRecipe): + version = '3.8.4' url = 'https://github.com/matplotlib/matplotlib/archive/v{version}.zip' - + patches = ["skip_macos.patch"] depends = ['kiwisolver', 'numpy', 'pillow', 'setuptools', 'freetype'] - python_depends = ['cycler', 'fonttools', 'packaging', 'pyparsing', 'python-dateutil'] + need_stl_shared = True def generate_libraries_pc_files(self, arch): """ @@ -62,8 +61,8 @@ def prebuild_arch(self, arch): ) self.generate_libraries_pc_files(arch) - def get_recipe_env(self, arch=None, with_flags_in_cc=True): - env = super().get_recipe_env(arch, with_flags_in_cc) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) # we make use of the same directory than `XDG_CACHE_HOME`, for our # custom library pc files, so we have all the install files that we diff --git a/pythonforandroid/recipes/matplotlib/skip_macos.patch b/pythonforandroid/recipes/matplotlib/skip_macos.patch new file mode 100644 index 0000000000..7652750769 --- /dev/null +++ b/pythonforandroid/recipes/matplotlib/skip_macos.patch @@ -0,0 +1,12 @@ +diff '--color=auto' -uNr matplotlib-3.8.4/setupext.py matplotlib-3.8.4.mod/setupext.py +--- matplotlib-3.8.4/setupext.py 2024-04-04 04:06:51.000000000 +0530 ++++ matplotlib-3.8.4.mod/setupext.py 2024-04-30 19:31:39.608063438 +0530 +@@ -782,7 +782,7 @@ + name = 'macosx' + + def check(self): +- if sys.platform != 'darwin': ++ if True: #sys.platform != 'darwin': + raise Skipped("Mac OS-X only") + return super().check() + diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 7e51eba66b..fb34c0c9f7 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -1,34 +1,29 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe -from pythonforandroid.logger import shprint, info -from pythonforandroid.util import current_directory -from multiprocessing import cpu_count +from pythonforandroid.recipe import Recipe, MesonRecipe +from pythonforandroid.logger import error from os.path import join -import glob -import sh import shutil +NUMPY_NDK_MESSAGE = "In order to build numpy, you must set minimum ndk api (minapi) to `24`.\n" -class NumpyRecipe(CompiledComponentsPythonRecipe): - version = '1.22.3' - url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' - site_packages_name = 'numpy' - depends = ["cython"] +class NumpyRecipe(MesonRecipe): + version = 'v1.26.5' + url = 'git+https://github.com/numpy/numpy' + hostpython_prerequisites = ["Cython>=3.0.6"] # meson does not detects venv's cython + extra_build_args = ['-Csetup-args=-Dblas=none', '-Csetup-args=-Dlapack=none'] + need_stl_shared = True - # This build specifically requires setuptools version 59.2.0 - hostpython_prerequisites = ["setuptools==59.2.0"] + def get_recipe_meson_options(self, arch): + options = super().get_recipe_meson_options(arch) + # Custom python is required, so that meson + # gets libs and config files properly + options["binaries"]["python"] = self.ctx.python_recipe.python_exe + options["binaries"]["python3"] = self.ctx.python_recipe.python_exe + options["properties"]["longdouble_format"] = "IEEE_DOUBLE_LE" if arch.arch in ["armeabi-v7a", "x86"] else "IEEE_QUAD_LE" + return options - install_in_hostpython = True - call_hostpython_via_targetpython = False - - patches = [ - join("patches", "remove-default-paths.patch"), - join("patches", "add_libm_explicitly_to_build.patch"), - join("patches", "ranlib.patch"), - ] - - def get_recipe_env(self, arch=None, with_flags_in_cc=True): - env = super().get_recipe_env(arch, with_flags_in_cc) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) # _PYTHON_HOST_PLATFORM declares that we're cross-compiling # and avoids issues when building on macOS for Android targets. @@ -37,54 +32,25 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): # NPY_DISABLE_SVML=1 allows numpy to build for non-AVX512 CPUs # See: https://github.com/numpy/numpy/issues/21196 env["NPY_DISABLE_SVML"] = "1" - + env["TARGET_PYTHON_EXE"] = join(Recipe.get_recipe( + "python3", self.ctx).get_build_dir(arch.arch), "android-build", "python") return env - def build_arch(self, arch): - self.hostpython_prerequisites = ["setuptools==59.2.0"] - self.install_hostpython_prerequisites() + def download_if_necessary(self): + # NumPy requires complex math functions which were added in api 24 + if self.ctx.ndk_api < 24: + error(NUMPY_NDK_MESSAGE) + exit(1) + super().download_if_necessary() + def build_arch(self, arch): super().build_arch(arch) - - # Post build step to restore setuptools version - self.hostpython_prerequisites = ["setuptools=={}".format( - Recipe.get_recipe("setuptools", self.ctx).version) - ] - self.install_hostpython_prerequisites() - - def _build_compiled_components(self, arch): - info('Building compiled components in {}'.format(self.name)) - - env = self.get_recipe_env(arch) - with current_directory(self.get_build_dir(arch.arch)): - hostpython = sh.Command(self.hostpython_location) - shprint(hostpython, 'setup.py', self.build_cmd, '-v', - _env=env, *self.setup_extra_args) - build_dir = glob.glob('build/lib.*')[0] - shprint(sh.find, build_dir, '-name', '"*.o"', '-exec', - env['STRIP'], '{}', ';', _env=env) - - def _rebuild_compiled_components(self, arch, env): - info('Rebuilding compiled components in {}'.format(self.name)) - - hostpython = sh.Command(self.real_hostpython_location) - shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env) - shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, - *self.setup_extra_args) - - def build_compiled_components(self, arch): - self.setup_extra_args = ['-j', str(cpu_count())] - self._build_compiled_components(arch) - self.setup_extra_args = [] - - def rebuild_compiled_components(self, arch, env): - self.setup_extra_args = ['-j', str(cpu_count())] - self._rebuild_compiled_components(arch, env) - self.setup_extra_args = [] + self.restore_hostpython_prerequisites(["cython"]) def get_hostrecipe_env(self, arch): env = super().get_hostrecipe_env(arch) env['RANLIB'] = shutil.which('ranlib') + env["LDFLAGS"] += " -lm" return env diff --git a/pythonforandroid/recipes/numpy/patches/add_libm_explicitly_to_build.patch b/pythonforandroid/recipes/numpy/patches/add_libm_explicitly_to_build.patch deleted file mode 100644 index f9ba9e924e..0000000000 --- a/pythonforandroid/recipes/numpy/patches/add_libm_explicitly_to_build.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py -index 66c07c9..d34bd93 100644 ---- a/numpy/linalg/setup.py -+++ b/numpy/linalg/setup.py -@@ -46,6 +46,7 @@ def configuration(parent_package='', top_path=None): - sources=['lapack_litemodule.c', get_lapack_lite_sources], - depends=['lapack_lite/f2c.h'], - extra_info=lapack_info, -+ libraries=['m'], - ) - - # umath_linalg module -@@ -54,7 +54,7 @@ def configuration(parent_package='', top_path=None): - sources=['umath_linalg.c.src', get_lapack_lite_sources], - depends=['lapack_lite/f2c.h'], - extra_info=lapack_info, -- libraries=['npymath'], -+ libraries=['npymath', 'm'], - ) - return config diff --git a/pythonforandroid/recipes/numpy/patches/ranlib.patch b/pythonforandroid/recipes/numpy/patches/ranlib.patch deleted file mode 100644 index c0b5dad6b4..0000000000 --- a/pythonforandroid/recipes/numpy/patches/ranlib.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -Naur numpy.orig/numpy/distutils/unixccompiler.py numpy/numpy/distutils/unixccompiler.py ---- numpy.orig/numpy/distutils/unixccompiler.py 2022-05-28 10:22:10.000000000 +0200 -+++ numpy/numpy/distutils/unixccompiler.py 2022-05-28 10:22:24.000000000 +0200 -@@ -124,6 +124,7 @@ - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. -+ self.ranlib = [os.environ.get('RANLIB')] - if self.ranlib: - display = '%s:@ %s' % (os.path.basename(self.ranlib[0]), - output_filename) diff --git a/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch b/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch deleted file mode 100644 index 3581f0f9ed..0000000000 --- a/pythonforandroid/recipes/numpy/patches/remove-default-paths.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py -index fc7018a..7b514bc 100644 ---- a/numpy/distutils/system_info.py -+++ b/numpy/distutils/system_info.py -@@ -340,10 +340,10 @@ if os.path.join(sys.prefix, 'lib') not in default_lib_dirs: - default_include_dirs.append(os.path.join(sys.prefix, 'include')) - default_src_dirs.append(os.path.join(sys.prefix, 'src')) - --default_lib_dirs = [_m for _m in default_lib_dirs if os.path.isdir(_m)] --default_runtime_dirs = [_m for _m in default_runtime_dirs if os.path.isdir(_m)] --default_include_dirs = [_m for _m in default_include_dirs if os.path.isdir(_m)] --default_src_dirs = [_m for _m in default_src_dirs if os.path.isdir(_m)] -+default_lib_dirs = [] #[_m for _m in default_lib_dirs if os.path.isdir(_m)] -+default_runtime_dirs =[] # [_m for _m in default_runtime_dirs if os.path.isdir(_m)] -+default_include_dirs =[] # [_m for _m in default_include_dirs if os.path.isdir(_m)] -+default_src_dirs =[] # [_m for _m in default_src_dirs if os.path.isdir(_m)] - - so_ext = get_shared_lib_extension() - -@@ -814,7 +814,7 @@ class system_info(object): - path = self.get_paths(self.section, key) - if path == ['']: - path = [] -- return path -+ return [] - - def get_include_dirs(self, key='include_dirs'): - return self.get_paths(self.section, key) diff --git a/pythonforandroid/recipes/pandas/__init__.py b/pythonforandroid/recipes/pandas/__init__.py index a43209a339..3f56adef6c 100644 --- a/pythonforandroid/recipes/pandas/__init__.py +++ b/pythonforandroid/recipes/pandas/__init__.py @@ -1,27 +1,25 @@ from os.path import join +from pythonforandroid.recipe import MesonRecipe -from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe - -class PandasRecipe(CppCompiledComponentsPythonRecipe): - version = '1.0.3' - url = 'https://github.com/pandas-dev/pandas/releases/download/v{version}/pandas-{version}.tar.gz' # noqa - - depends = ['cython', 'numpy', 'libbz2', 'liblzma'] - - python_depends = ['python-dateutil', 'pytz'] +class PandasRecipe(MesonRecipe): + version = 'v2.2.1' + url = 'git+https://github.com/pandas-dev/pandas' # noqa + depends = ['numpy', 'libbz2', 'liblzma'] + hostpython_prerequisites = ["Cython~=3.0.5"] # meson does not detects venv's cython patches = ['fix_numpy_includes.patch'] - - call_hostpython_via_targetpython = False + python_depends = ['python-dateutil', 'pytz'] need_stl_shared = True - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) # we need the includes from our installed numpy at site packages # because we need some includes generated at numpy's compile time + env['NUMPY_INCLUDES'] = join( self.ctx.get_python_install_dir(arch.arch), "numpy/core/include", ) + env["PYTHON_INCLUDE_DIR"] = self.ctx.python_recipe.include_root(arch) # this flag below is to fix a runtime error: # ImportError: dlopen failed: cannot locate symbol @@ -31,5 +29,9 @@ def get_recipe_env(self, arch): env['LDFLAGS'] += f' -landroid -l{self.stl_lib_name}' return env + def build_arch(self, arch): + super().build_arch(arch) + self.restore_hostpython_prerequisites(["cython"]) + recipe = PandasRecipe() diff --git a/pythonforandroid/recipes/pandas/fix_numpy_includes.patch b/pythonforandroid/recipes/pandas/fix_numpy_includes.patch index ef1643b9b1..5c84e6421b 100644 --- a/pythonforandroid/recipes/pandas/fix_numpy_includes.patch +++ b/pythonforandroid/recipes/pandas/fix_numpy_includes.patch @@ -1,31 +1,82 @@ ---- pandas-1.0.1/setup.py.orig 2020-02-05 17:15:24.000000000 +0100 -+++ pandas-1.0.1/setup.py 2020-03-15 13:47:57.612237225 +0100 -@@ -37,11 +37,12 @@ min_cython_ver = "0.29.13" # note: sync - - setuptools_kwargs = { - "install_requires": [ -- "python-dateutil >= 2.6.1", -- "pytz >= 2017.2", -- f"numpy >= {min_numpy_ver}", -+ # dependencies managed via p4a's recipe -+ # "python-dateutil >= 2.6.1", -+ # "pytz >= 2017.2", -+ # f"numpy >= {min_numpy_ver}", - ], -- "setup_requires": [f"numpy >= {min_numpy_ver}"], -+ "setup_requires": [], - "zip_safe": False, - } - -@@ -514,7 +515,10 @@ def maybe_cythonize(extensions, *args, * - ) - raise RuntimeError("Cannot cythonize without Cython installed.") +diff '--color=auto' -uNr pandas/pandas/_libs/meson.build pandas.mod/pandas/_libs/meson.build +--- pandas/pandas/_libs/meson.build 2024-04-24 07:24:46.009296003 +0530 ++++ pandas.mod/pandas/_libs/meson.build 2024-04-24 07:45:15.221534571 +0530 +@@ -115,7 +115,7 @@ + ext_name, + ext_dict.get('sources'), + cython_args: cython_args, +- include_directories: [inc_np, inc_pd], ++ include_directories: [inc_android, inc_np, inc_pd], + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs', + install: true +diff '--color=auto' -uNr pandas/pandas/_libs/tslibs/meson.build pandas.mod/pandas/_libs/tslibs/meson.build +--- pandas/pandas/_libs/tslibs/meson.build 2024-04-24 07:24:46.019296090 +0530 ++++ pandas.mod/pandas/_libs/tslibs/meson.build 2024-04-24 07:45:53.528798309 +0530 +@@ -33,7 +33,7 @@ + ext_name, + ext_dict.get('sources'), + cython_args: cython_args, +- include_directories: [inc_np, inc_pd], ++ include_directories: [inc_android, inc_np, inc_pd], + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs/tslibs', + install: true +diff '--color=auto' -uNr pandas/pandas/_libs/window/meson.build pandas.mod/pandas/_libs/window/meson.build +--- pandas/pandas/_libs/window/meson.build 2024-04-24 07:24:46.029296177 +0530 ++++ pandas.mod/pandas/_libs/window/meson.build 2024-04-28 10:47:16.915307381 +0530 +@@ -2,7 +2,7 @@ + 'aggregations', + ['aggregations.pyx'], + cython_args: ['-X always_allow_keywords=true'], +- include_directories: [inc_np, inc_pd], ++ include_directories: [inc_android, inc_np, inc_pd], + subdir: 'pandas/_libs/window', + override_options : ['cython_language=cpp'], + install: true +@@ -12,7 +12,7 @@ + 'indexers', + ['indexers.pyx'], + cython_args: ['-X always_allow_keywords=true'], +- include_directories: [inc_np, inc_pd], ++ include_directories: [inc_android, inc_np, inc_pd], + subdir: 'pandas/_libs/window', + install: true + ) +diff '--color=auto' -uNr pandas/pandas/meson.build pandas.mod/pandas/meson.build +--- pandas/pandas/meson.build 2024-04-24 07:24:46.232297943 +0530 ++++ pandas.mod/pandas/meson.build 2024-04-24 07:46:12.508929590 +0530 +@@ -3,20 +3,23 @@ + '-c', + ''' + import os +-import numpy as np +-try: +- # Check if include directory is inside the pandas dir +- # e.g. a venv created inside the pandas dir +- # If so, convert it to a relative path +- incdir = os.path.relpath(np.get_include()) +-except Exception: +- incdir = np.get_include() +-print(incdir) +- ''' ++print(os.environ["NUMPY_INCLUDES"]) ++ ''' ++ ], ++ check: true ++).stdout().strip() ++incdir_android = run_command(py, ++ [ ++ '-c', ++ ''' ++import os ++print(os.environ["PYTHON_INCLUDE_DIR"]) ++ ''' + ], + check: true + ).stdout().strip() + ++inc_android = include_directories(incdir_android) + inc_np = include_directories(incdir_numpy) + inc_pd = include_directories('_libs/include') -- numpy_incl = pkg_resources.resource_filename("numpy", "core/include") -+ if 'NUMPY_INCLUDES' in os.environ: -+ numpy_incl = os.environ['NUMPY_INCLUDES'] -+ else: -+ numpy_incl = pkg_resources.resource_filename("numpy", "core/include") - # TODO: Is this really necessary here? - for ext in extensions: - if hasattr(ext, "include_dirs") and numpy_incl not in ext.include_dirs: diff --git a/pythonforandroid/recipes/pydantic-core/__init__.py b/pythonforandroid/recipes/pydantic-core/__init__.py index 8702dfc002..bf76a65d0a 100644 --- a/pythonforandroid/recipes/pydantic-core/__init__.py +++ b/pythonforandroid/recipes/pydantic-core/__init__.py @@ -4,8 +4,6 @@ class PydanticcoreRecipe(RustCompiledComponentsRecipe): version = "2.16.1" url = "https://github.com/pydantic/pydantic-core/archive/refs/tags/v{version}.tar.gz" - use_maturin = True - hostpython_prerequisites = ["typing_extensions"] site_packages_name = "pydantic_core" diff --git a/pythonforandroid/recipes/scipy/__init__.py b/pythonforandroid/recipes/scipy/__init__.py index bde9758d8d..242ca04234 100644 --- a/pythonforandroid/recipes/scipy/__init__.py +++ b/pythonforandroid/recipes/scipy/__init__.py @@ -19,6 +19,7 @@ class ScipyRecipe(CompiledComponentsPythonRecipe): url = 'git+https://github.com/scipy/scipy.git' git_commit = 'b430bf54b5064465983813e2cfef3fcb86c3df07' # version 1.11.3 site_packages_name = 'scipy' + hostpython_prerequisites = ['numpy'] depends = ['setuptools', 'cython', 'numpy', 'lapack', 'pybind11'] call_hostpython_via_targetpython = False need_stl_shared = True diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index cbcfdd2b6e..269a57fcf8 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -163,7 +163,7 @@ def check_target_api(api, arch): MIN_NDK_API = 21 -RECOMMENDED_NDK_API = 21 +RECOMMENDED_NDK_API = 24 OLD_NDK_API_MESSAGE = ('NDK API less than {} is not supported'.format(MIN_NDK_API)) TARGET_NDK_API_GREATER_THAN_TARGET_API_MESSAGE = ( 'Target NDK API is {ndk_api}, ' diff --git a/setup.py b/setup.py index badce08e82..4a628df905 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'sh>=1.10, <2.0; sys_platform!="win32"', - 'build', 'toml', 'packaging', 'setuptools' + 'build', 'toml', 'packaging', 'setuptools', 'wheel~=0.43.0' ] # (build and toml are used by pythonpackage.py) diff --git a/testapps/on_device_unit_tests/setup.py b/testapps/on_device_unit_tests/setup.py index 18fd82af75..a63ca8bcb7 100644 --- a/testapps/on_device_unit_tests/setup.py +++ b/testapps/on_device_unit_tests/setup.py @@ -43,7 +43,7 @@ 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' 'chardet,idna', 'android-api': 27, - 'ndk-api': 21, + 'ndk-api': 24, 'dist-name': 'bdist_unit_tests_app', 'arch': 'armeabi-v7a', 'bootstrap' : 'sdl2', @@ -57,7 +57,7 @@ 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' 'chardet,idna', 'android-api': 27, - 'ndk-api': 21, + 'ndk-api': 24, 'dist-name': 'bdist_unit_tests_app', 'arch': 'armeabi-v7a', 'bootstrap' : 'sdl2', @@ -69,7 +69,7 @@ { 'requirements' : 'python3', 'android-api': 27, - 'ndk-api': 21, + 'ndk-api': 24, 'dist-name': 'bdist_unit_tests_app', 'arch': 'arm64-v8a', 'bootstrap' : 'service_library',