From 1236ea3a0215e95823a70d83fa9107b38a64f2aa Mon Sep 17 00:00:00 2001 From: NHellFire Date: Sun, 22 Jan 2023 19:59:31 +0000 Subject: [PATCH 1/4] Update libraries to versions with arm64 wheels --- embedding-calculator/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/embedding-calculator/requirements.txt b/embedding-calculator/requirements.txt index d669ae1bec..e087e9ec05 100644 --- a/embedding-calculator/requirements.txt +++ b/embedding-calculator/requirements.txt @@ -21,9 +21,9 @@ Pillow~=9.2.0 imagecodecs~=2020.5.30 numpy~=1.21 scipy~=1.5.4 -opencv-python~=4.4.0 -scikit-learn~=0.23.2 -scikit-image~=0.17.2 +opencv-python~=4.5.0 +scikit-learn~=0.24.2 +scikit-image~=0.18.3 joblib~=1.2.0 # web server From ca562388e44756bcc55fd4bc6b0a229e31c1de4d Mon Sep 17 00:00:00 2001 From: NHellFire Date: Sun, 22 Jan 2023 20:01:43 +0000 Subject: [PATCH 2/4] Build imagecodecs from source (no arm64 wheels available) --- embedding-calculator/Dockerfile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/embedding-calculator/Dockerfile b/embedding-calculator/Dockerfile index 63c2182a97..e123b4a2c9 100644 --- a/embedding-calculator/Dockerfile +++ b/embedding-calculator/Dockerfile @@ -4,12 +4,45 @@ FROM ${BASE_IMAGE:-python:3.7-slim} RUN apt-get update && apt-get install -y build-essential cmake git wget unzip \ curl yasm pkg-config libswscale-dev libtbb2 libtbb-dev libjpeg-dev \ libpng-dev libtiff-dev libavformat-dev libpq-dev libfreeimage3 python3-opencv \ + libaec-dev libblosc-dev libbrotli-dev libbz2-dev libgif-dev libopenjp2-7-dev \ + liblcms2-dev libcharls-dev libjxr-dev liblz4-dev libcfitsio-dev libsnappy-dev \ + libwebp-dev libzopfli-dev libzstd-dev \ && rm -rf /var/lib/apt/lists/* +# Dependencies for imagecodecs +WORKDIR /tmp + +# brunsli +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.1 https://github.com/google/brunsli && \ + cd brunsli && \ + cmake -DCMAKE_BUILD_TYPE=Release . && \ + make -j$(nproc) install && \ + rm -rf /tmp/brunsli + +# libjxl +RUN git clone --depth=1 --shallow-submodules --recursive -b v0.7.0 https://github.com/libjxl/libjxl && \ + cd libjxl && \ + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF . && \ + make -j$(nproc) install && \ + rm -rf /tmp/libjxl + +# zfp +RUN git clone --depth=1 -b 0.5.5 https://github.com/LLNL/zfp && \ + cd zfp && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ + make -j$(nproc) install && \ + rm -rf /tmp/zfp +# End imagecodecs dependencies + # install common python packages SHELL ["/bin/bash", "-c"] WORKDIR /app/ml COPY requirements.txt . +# Ensure numpy is installed first as imagecodecs doesn't declare dependencies correctly until 2022.9.26, +# which is not compatible with Python 3.7 +RUN pip --no-cache-dir install $(grep ^numpy requirements.txt) RUN pip --no-cache-dir install -r requirements.txt ARG BE_VERSION From 307d923ad3004ede2da644f08963477b0f699613 Mon Sep 17 00:00:00 2001 From: NHellFire Date: Sun, 22 Jan 2023 20:23:40 +0000 Subject: [PATCH 3/4] Update imagecodecs so it can find openjpeg 2.4 --- embedding-calculator/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedding-calculator/requirements.txt b/embedding-calculator/requirements.txt index e087e9ec05..976706fca4 100644 --- a/embedding-calculator/requirements.txt +++ b/embedding-calculator/requirements.txt @@ -18,7 +18,7 @@ pylama~=7.7.1 # dependencies for both scanner backends Pillow~=9.2.0 -imagecodecs~=2020.5.30 +imagecodecs~=2021.11.20 numpy~=1.21 scipy~=1.5.4 opencv-python~=4.5.0 From 43b4953953a0d51a1721278aa826be02858d30f0 Mon Sep 17 00:00:00 2001 From: NHellFire Date: Mon, 23 Jan 2023 09:10:14 +0000 Subject: [PATCH 4/4] Update tensorflow so we have arm64 wheels Also downgrade protobuf to meet tensorflow's deps --- .../src/services/facescan/plugins/agegender/__init__.py | 2 +- .../src/services/facescan/plugins/dependencies.py | 2 +- .../src/services/facescan/plugins/facenet/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py b/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py index dccb4a7368..77306a2dca 100644 --- a/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py +++ b/embedding-calculator/src/services/facescan/plugins/agegender/__init__.py @@ -12,4 +12,4 @@ # or implied. See the License for the specific language governing # permissions and limitations under the License. -requirements = ('tensorflow~=2.1.4', 'tf-slim~=1.1.0') +requirements = ('tensorflow~=2.11.0', 'tf-slim~=1.1.0') diff --git a/embedding-calculator/src/services/facescan/plugins/dependencies.py b/embedding-calculator/src/services/facescan/plugins/dependencies.py index f5ff371f1b..c238a82c94 100644 --- a/embedding-calculator/src/services/facescan/plugins/dependencies.py +++ b/embedding-calculator/src/services/facescan/plugins/dependencies.py @@ -18,7 +18,7 @@ from src.services.utils.pyutils import get_env -def get_tensorflow(version='2.1.4') -> Tuple[str, ...]: +def get_tensorflow(version='2.11.0') -> Tuple[str, ...]: libs = [f'tensorflow=={version}'] cuda_version = get_env('CUDA', '').replace('.', '') if ENV.GPU_IDX > -1 and cuda_version: diff --git a/embedding-calculator/src/services/facescan/plugins/facenet/__init__.py b/embedding-calculator/src/services/facescan/plugins/facenet/__init__.py index 0171a996e6..0c0ab221e7 100644 --- a/embedding-calculator/src/services/facescan/plugins/facenet/__init__.py +++ b/embedding-calculator/src/services/facescan/plugins/facenet/__init__.py @@ -14,4 +14,4 @@ from src.services.facescan.plugins.dependencies import get_tensorflow -requirements = ('protobuf~=3.20.1',) + get_tensorflow() # + ('mtcnn~=0.1.0',) +requirements = ('protobuf~=3.19.6',) + get_tensorflow() # + ('mtcnn~=0.1.0',)