From 91b128bcd347855f8aa94014556273bfb9f9f467 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 15 Jul 2020 16:35:16 -0700 Subject: [PATCH 01/15] Linking Python Include Dirs in the build for Python 3.6, Python 3.7 --- scripts/install_python_toolchain.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index fe4e983661..2e9919e421 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -6,7 +6,8 @@ if [[ -z $VIRTUALENV ]]; then VIRTUALENV=virtualenv fi -$VIRTUALENV "$(pwd)"/deps/env +DEPS_ENV_FOLDER="$(pwd)"/deps/env +$VIRTUALENV $DEPS_ENV_FOLDER source deps/env/bin/activate PYTHON="${PWD}/deps/env/bin/python" @@ -14,6 +15,7 @@ PIP="${PYTHON} -m pip" PYTHON_MAJOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.major)') PYTHON_MINOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.minor)') +PYTHON_INCLUDE_DIR=$(${PYTHON} -c 'from sysconfig import get_paths as gp; print(gp()["include"])') PYTHON_VERSION="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}" # TODO - not sure why 'm' is necessary here (and not in 2.7) @@ -23,6 +25,9 @@ if [[ "${PYTHON_VERSION}" == "python2.7" ]]; then PYTHON_FULL_NAME=python2.7 fi +DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include +mkdir -p $DEPS_INCLUDE_FOLDER +ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER"/"$PYTHON_FULL_NAME" function linux_patch_sigfpe_handler { if [[ $OSTYPE == linux* ]]; then From 371aa55ae44a636998fd3e29fe4bf3faed1d02f7 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 15 Jul 2020 18:21:03 -0700 Subject: [PATCH 02/15] To prevent recursive symlinks --- scripts/install_python_toolchain.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 2e9919e421..3a7e3f2dee 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -26,6 +26,7 @@ if [[ "${PYTHON_VERSION}" == "python2.7" ]]; then fi DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include +rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER"/"$PYTHON_FULL_NAME" From 72fc566ec6e8b55010d9d7d09c141754cb7db4c7 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Thu, 16 Jul 2020 15:12:34 -0700 Subject: [PATCH 03/15] Added check for python 3.7 --- scripts/install_python_toolchain.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 3a7e3f2dee..9cbd7f5475 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -25,10 +25,12 @@ if [[ "${PYTHON_VERSION}" == "python2.7" ]]; then PYTHON_FULL_NAME=python2.7 fi -DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include -rm -rf $DEPS_INCLUDE_FOLDER -mkdir -p $DEPS_INCLUDE_FOLDER -ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER"/"$PYTHON_FULL_NAME" +if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then + DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include + rm -rf $DEPS_INCLUDE_FOLDER + mkdir -p $DEPS_INCLUDE_FOLDER + ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER"/"$PYTHON_FULL_NAME" +fi function linux_patch_sigfpe_handler { if [[ $OSTYPE == linux* ]]; then From 655877d543ac6934abb803b89b9f1a47e376552f Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Tue, 21 Jul 2020 13:24:36 -0700 Subject: [PATCH 04/15] Updated Python Include script --- scripts/install_python_toolchain.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 9cbd7f5475..a0090271e5 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -15,7 +15,7 @@ PIP="${PYTHON} -m pip" PYTHON_MAJOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.major)') PYTHON_MINOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.minor)') -PYTHON_INCLUDE_DIR=$(${PYTHON} -c 'from sysconfig import get_paths as gp; print(gp()["include"])') +PYTHON_INCLUDE_DIR=$(${PYTHON} -c 'from sysconfig import get_paths as gp; print("/".join(gp()["include"].split("/")[:-1]))') PYTHON_VERSION="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}" # TODO - not sure why 'm' is necessary here (and not in 2.7) @@ -29,7 +29,7 @@ if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER - ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER"/"$PYTHON_FULL_NAME" + ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER" fi function linux_patch_sigfpe_handler { From ab9f4498e9954bd8d61d86001cebd2ba9eb30932 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Tue, 21 Jul 2020 15:59:04 -0700 Subject: [PATCH 05/15] Updated linking script for python --- scripts/install_python_toolchain.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index a0090271e5..25e3a22808 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -27,9 +27,10 @@ fi if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include + PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER - ln -sfn "$PYTHON_INCLUDE_DIR" "$DEPS_INCLUDE_FOLDER" + ln -sfn $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER fi function linux_patch_sigfpe_handler { From 16264cf0a1f8cac82174949b837e62da2404ff78 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 09:35:09 -0700 Subject: [PATCH 06/15] Added Python Toolchain --- scripts/install_python_toolchain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 25e3a22808..309a4042e5 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -30,7 +30,7 @@ if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER - ln -sfn $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER + cp -r $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER fi function linux_patch_sigfpe_handler { From ef42319ce2b6e2ca0a38bed8bdbbbb17dc280924 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 09:59:50 -0700 Subject: [PATCH 07/15] Debugging --- scripts/install_python_toolchain.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 309a4042e5..101ec4a5b1 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -30,6 +30,7 @@ if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER + ls $PYTHON_INCLUDE_DIR cp -r $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER fi From 7aec78cdb508c357652215abdd29a3d182fb70f0 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 12:23:57 -0700 Subject: [PATCH 08/15] Debugging upstream --- scripts/install_python_toolchain.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 101ec4a5b1..105f22ec27 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -28,6 +28,7 @@ fi if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' + ls $PYTHON_INCLUDE_DIR rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER ls $PYTHON_INCLUDE_DIR From e8bb2520acdbdf74e7b7e4059597494f427e47e9 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 12:29:50 -0700 Subject: [PATCH 09/15] If the operation is already complete don't link headers --- scripts/install_python_toolchain.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 105f22ec27..42bae45f12 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -28,11 +28,11 @@ fi if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' - ls $PYTHON_INCLUDE_DIR - rm -rf $DEPS_INCLUDE_FOLDER - mkdir -p $DEPS_INCLUDE_FOLDER - ls $PYTHON_INCLUDE_DIR - cp -r $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER + if [[ ${DEPS_INCLUDE_FOLDER} -ne ${PYTHON_INCLUDE_DIR} ]]; then + rm -rf $DEPS_INCLUDE_FOLDER + mkdir -p $DEPS_INCLUDE_FOLDER + cp -r $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER + fi fi function linux_patch_sigfpe_handler { From 7f06678dd6750f48ecb18008fe95d0d5e710405c Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 15:49:26 -0700 Subject: [PATCH 10/15] Updated requirements.txt for test failures --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 07587ab682..6d176b8d9c 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -11,7 +11,7 @@ tensorflow==2.0.1; python_version > '2.7' tensorflow==2.0.0; python_version < '3.5' # Last version to support Python 2.7 UISoup==2.5.7 pyobjc==5.2; sys_platform == 'darwin' -llvmlite==0.31.0 +llvmlite==0.33.0 ########## >= ######### From 776916f0ba5c2deeef0bad3abd01e5bba7554b1d Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 22 Jul 2020 18:21:07 -0700 Subject: [PATCH 11/15] Updated LLVM lite --- scripts/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 6d176b8d9c..579152347d 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -11,7 +11,8 @@ tensorflow==2.0.1; python_version > '2.7' tensorflow==2.0.0; python_version < '3.5' # Last version to support Python 2.7 UISoup==2.5.7 pyobjc==5.2; sys_platform == 'darwin' -llvmlite==0.33.0 +llvmlite==0.33.0; python_version == '3.8' +llvmlite==0.31.0; python_version != '3.8' ########## >= ######### From 8c0efc1e65c73ce660e3bd8b416daa81e55a2f96 Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Mon, 27 Jul 2020 09:49:16 -0700 Subject: [PATCH 12/15] Udpated Requirements.txt --- scripts/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 579152347d..ae2093972f 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -11,8 +11,8 @@ tensorflow==2.0.1; python_version > '2.7' tensorflow==2.0.0; python_version < '3.5' # Last version to support Python 2.7 UISoup==2.5.7 pyobjc==5.2; sys_platform == 'darwin' -llvmlite==0.33.0; python_version == '3.8' -llvmlite==0.31.0; python_version != '3.8' +llvmlite==0.33.0; python_version > '2.7' +llvmlite==0.31.0; python_version < '3.5' ########## >= ######### From 843b0adb855f1a4b262302e2f9e644f6b51c417f Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Mon, 27 Jul 2020 11:09:41 -0700 Subject: [PATCH 13/15] Explicitly find Python toolchain --- scripts/install_python_toolchain.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index 42bae45f12..ba75053db7 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -25,10 +25,17 @@ if [[ "${PYTHON_VERSION}" == "python2.7" ]]; then PYTHON_FULL_NAME=python2.7 fi +# For robustness on different system, different installations, +# actually locate Python.h before setting it as an include +# +# Recursively find Python.h and then parse the include directory +PYTHON_HEADER_PATH=$(find ${PYTHON_INCLUDE_DIR} -name "python*" | head -n 1) +PYTHON_PATH=${PYTHON_HEADER_PATH%include/*} +PYTHON_INCLUDE_PATH="${PYTHON_PATH}include/" + if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include - PYTHON_INCLUDE_PATH="$PYTHON_INCLUDE_DIR"/python'*' - if [[ ${DEPS_INCLUDE_FOLDER} -ne ${PYTHON_INCLUDE_DIR} ]]; then + if [[ ${DEPS_INCLUDE_FOLDER} != ${PYTHON_INCLUDE_DIR} ]]; then rm -rf $DEPS_INCLUDE_FOLDER mkdir -p $DEPS_INCLUDE_FOLDER cp -r $PYTHON_INCLUDE_PATH $DEPS_INCLUDE_FOLDER From 61b7dd7c2aa2ca427d5ce1be269d53ae12bfba7b Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Mon, 27 Jul 2020 11:28:08 -0700 Subject: [PATCH 14/15] Added the python* to the Include Path --- scripts/install_python_toolchain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index ba75053db7..c476ffe34a 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -31,7 +31,7 @@ fi # Recursively find Python.h and then parse the include directory PYTHON_HEADER_PATH=$(find ${PYTHON_INCLUDE_DIR} -name "python*" | head -n 1) PYTHON_PATH=${PYTHON_HEADER_PATH%include/*} -PYTHON_INCLUDE_PATH="${PYTHON_PATH}include/" +PYTHON_INCLUDE_PATH="${PYTHON_PATH}include/python*" if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include From a2abce268fe22e536ea7d2330d2430609496e8ae Mon Sep 17 00:00:00 2001 From: Abhishek Pratapa Date: Wed, 29 Jul 2020 13:00:40 -0700 Subject: [PATCH 15/15] enclosed find script behind python check --- scripts/install_python_toolchain.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh index c476ffe34a..683654a1b1 100755 --- a/scripts/install_python_toolchain.sh +++ b/scripts/install_python_toolchain.sh @@ -29,11 +29,10 @@ fi # actually locate Python.h before setting it as an include # # Recursively find Python.h and then parse the include directory -PYTHON_HEADER_PATH=$(find ${PYTHON_INCLUDE_DIR} -name "python*" | head -n 1) -PYTHON_PATH=${PYTHON_HEADER_PATH%include/*} -PYTHON_INCLUDE_PATH="${PYTHON_PATH}include/python*" - if [[ ${PYTHON_MAJOR_VERSION} -ge 3 && ${PYTHON_MINOR_VERSION} -gt 5 ]]; then + PYTHON_HEADER_PATH=$(find ${PYTHON_INCLUDE_DIR} -name "python*" | head -n 1) + PYTHON_PATH=${PYTHON_HEADER_PATH%include/*} + PYTHON_INCLUDE_PATH="${PYTHON_PATH}include/python*" DEPS_INCLUDE_FOLDER="$DEPS_ENV_FOLDER"/include if [[ ${DEPS_INCLUDE_FOLDER} != ${PYTHON_INCLUDE_DIR} ]]; then rm -rf $DEPS_INCLUDE_FOLDER