feat: update mlflow version to support newer version of protobuf (#465)

**What this PR does / why we need it**:  * Mlflow 1.26.1 or above is required in order to be compatible with newer version of protobuf. * This PR also drops the support for Python 3.7 from the real-time and batch predictor servers (continuing from #464). Docs and examples are updated accordingly. **Which issue(s) this PR fixes**:  Fixes # **Does this PR introduce a user-facing change?**:  ```release-note Upgrade MlFlow client version; drop Python 3.7 support ``` **Checklist** - [ ] Added unit test, integration, and/or e2e tests - [ ] Tested locally - [x] Updated documentation - [ ] Update Swagger spec if the PR introduce API changes - [ ] Regenerated Golang and Python client if the PR introduce API changes --------- Co-authored-by: Krithika Sundararajan <[email protected]>
caraml-dev · Nov 2, 2023 · 32ba4d6 · 32ba4d6
1 parent 36993fc
commit 32ba4d6
Show file tree

Hide file tree

Showing 31 changed files with 52 additions and 77 deletions.
diff --git a/.github/workflows/merlin.yml b/.github/workflows/merlin.yml
@@ -42,7 +42,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"] #TODO: Remove Python 3.7 support
+        python-version: ["3.8", "3.9", "3.10"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
     steps:
@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"] #TODO: Remove Python 3.7 support
+        python-version: ["3.8", "3.9", "3.10"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
     steps:
@@ -263,7 +263,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["37", "38", "39", "310"]
+        python-version: ["38", "39", "310"]
     needs:
       - create-version
     steps:
@@ -284,7 +284,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["37", "38", "39", "310"]
+        python-version: ["38", "39", "310"]
     needs:
       - create-version
     steps:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -63,8 +63,7 @@ jobs:
         working-directory: ./python/sdk
         run: |
           IMAGE_TAG="${{ env.CONTAINER_REGISTRY }}/merlin-sdk:py${{ matrix.python-version }}-${{ inputs.version }}"
-          SDK_VERSION="$(sed 's/^v\(.*\)-/\1/' <<< "${{ inputs.version }}")"
-          docker build -t ${IMAGE_TAG} --build-arg PYTHON_VERSION=${{ matrix.python-version }} --build-arg VERSION=${SDK_VERSION} .
+          docker build -t ${IMAGE_TAG} --build-arg PYTHON_VERSION=${{ matrix.python-version }} .
           docker push ${IMAGE_TAG}
 
   publish-api:
@@ -106,7 +105,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["37", "38", "39", "310"]
+        python-version: ["38", "39", "310"]
     steps:
       - name: Log in to the Container registry
         uses: docker/login-action@v1
@@ -132,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["37", "38", "39", "310"]
+        python-version: ["38", "39", "310"]
     steps:
       - name: Log in to the Container registry
         uses: docker/login-action@v1

diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@ TEST_TAGS?=
 GOLANGCI_LINT_VERSION="v1.51.2"
 PROTOC_GEN_GO_JSON_VERSION="v1.1.0"
 PROTOC_GEN_GO_VERSION="v1.26"
-PYTHON_VERSION ?= "37"	#set as 37 38 39 310 for 3.7-3.10 respectively
+PYTHON_VERSION ?= "39"	#set as 38 39 310 for 3.8-3.10 respectively
 
 all: setup init-dep lint test clean build run
 

diff --git a/examples/batch/env.yaml b/examples/batch/env.yaml
@@ -2,6 +2,6 @@ dependencies:
 - pip:
     - joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
     - numpy>=1.19.5
-    - scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+    - scikit-learn>=1.1.2
     - xgboost==1.6.2
-    - mlflow==1.23.0
+    - mlflow==1.26.1
diff --git a/examples/batch/requirements.txt b/examples/batch/requirements.txt
@@ -1,8 +1,8 @@
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 numpy>=1.19.5
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 xgboost==1.6.2
-mlflow==1.23.0
+mlflow==1.26.1
 merlin-sdk
 cloudpickle==2.0.0
 google-cloud

diff --git a/examples/custom-model/http_json/requirements.txt b/examples/custom-model/http_json/requirements.txt
@@ -1,3 +1,3 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 xgboost==1.6.2
 merlin-sdk
diff --git a/examples/model-endpoint/requirements.txt b/examples/model-endpoint/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 merlin-sdk
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 cloudpickle==2.0.0
diff --git a/examples/pyfunc/env.yaml b/examples/pyfunc/env.yaml
@@ -2,5 +2,5 @@ dependencies:
   - python=3.8
   - pip:
       - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version
-      - scikit-learn==1.1.2 #TODO: >=1.1.2 upon python 3.7 deprecation
+      - scikit-learn>=1.1.2
       - xgboost==1.6.2
diff --git a/examples/pyfunc/requirements.txt b/examples/pyfunc/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 xgboost==1.6.2
 merlin-sdk

diff --git a/examples/resource-request-gpu/requirements.txt b/examples/resource-request-gpu/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 merlin-sdk
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 cloudpickle==2.0.0
diff --git a/examples/resource-request/requirements.txt b/examples/resource-request/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 merlin-sdk
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 cloudpickle==2.0.0
diff --git a/examples/sklearn/requirements.txt b/examples/sklearn/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 merlin-sdk
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 cloudpickle==2.0.0
diff --git a/examples/xgboost/requirements.txt b/examples/xgboost/requirements.txt
@@ -1,4 +1,4 @@
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 xgboost==1.6.2
 merlin-sdk
 cloudpickle==2.0.0
diff --git a/python/batch-predictor/README.md b/python/batch-predictor/README.md
@@ -69,7 +69,7 @@ OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python main.py --job-name iris-predictio
 
 ### Requirements
 
-- python 3.7.0
+- python >= 3.8.0
 - pipenv (install using `pip install pipenv`)
 - protoc (see [installation instruction](http://google.github.io/proto-lens/installing-protoc.html))
 - gcloud (see [installation instruction](https://cloud.google.com/sdk/install))

diff --git a/python/batch-predictor/docker/app.Dockerfile b/python/batch-predictor/docker/app.Dockerfile
@@ -24,6 +24,6 @@ ARG GOOGLE_APPLICATION_CREDENTIALS
 RUN if [[ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}; fi
 RUN gsutil -m cp -r ${MODEL_URL} .
 RUN /bin/bash -c ". activate ${CONDA_ENVIRONMENT} && \
-    sed -i 's/mlflow$/mlflow==1.23.0/' ${HOME}/model/conda.yaml && \
+    sed -i 's/mlflow\(\s*==\s*[^ ]*\)\{0,1\}/mlflow==1.26.1/g' ${HOME}/model/conda.yaml && \
     conda env update --name ${CONDA_ENVIRONMENT} --file ${HOME}/model/conda.yaml && \
     python ${HOME}/merlin-spark-app/main.py --dry-run-model ${HOME}/model"
diff --git a/python/batch-predictor/docker/base.Dockerfile b/python/batch-predictor/docker/base.Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gcr.io/spark-operator/spark-py:v3.0.0
+FROM apache/spark-py:v3.1.3
 
 # Switch to user root so we can add additional jars and configuration files.
 USER root

diff --git a/python/batch-predictor/docker/env37.yaml b/python/batch-predictor/docker/env37.yaml
diff --git a/python/batch-predictor/merlinpyspark/model.py b/python/batch-predictor/merlinpyspark/model.py
@@ -67,7 +67,7 @@ def spark_udf(spark, model_uri, features, result_type="double"):
         archive_path = SparkModelCache.add_local_model(spark, local_model_path)
 
     def predict(*args):
-        model = SparkModelCache.get_or_load(archive_path)
+        model, _ = SparkModelCache.get_or_load(archive_path)
         schema = {features[i]: arg for i, arg in enumerate(args)}
         pdf = None
         for x in args:

diff --git a/python/batch-predictor/requirements.txt b/python/batch-predictor/requirements.txt
@@ -1,8 +1,7 @@
 findspark
 pyspark==3.0.1
-mlflow>=1.2.0,<=1.23.0
+mlflow>=1.26.1,<2.0.0
 cloudpickle==2.0.0
 pyarrow>=0.14.1,<=9.0.0
-protobuf>=3.0,<4.0.0
-#TODO: Update merlin-sdk dep to: file:${SDK_PATH}#egg=merlin-sdk
-merlin-sdk==0.33.0 # Pin to the version that supports Python 3.7.
+protobuf>=3.0,<5.0.0
+file:${SDK_PATH}#egg=merlin-sdk
diff --git a/python/batch-predictor/requirements_test.txt b/python/batch-predictor/requirements_test.txt
@@ -2,9 +2,7 @@ pytest
 pytest-cov
 mypy
 google-cloud-bigquery
-scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+scikit-learn>=1.1.2
 joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
 mypy-protobuf>=1.19
-types-PyYAML
-protobuf<4.0.0
-grpcio<1.49.0
+types-PyYAML
diff --git a/python/batch-predictor/setup.py b/python/batch-predictor/setup.py
@@ -21,20 +21,19 @@
 with open('requirements.txt') as f:
     REQUIRE = f.read().splitlines()
 
-# TODO: Uncomment below lines after Pyfunc server stops supporting Python 3.7
-# merlin_path = os.path.join(os.getcwd(), "../sdk")
-# merlin_sdk_package = "merlin-sdk"
-# for index, item in enumerate(REQUIRE):
-#     if merlin_sdk_package in item:
-#         REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}"
+merlin_path = os.path.join(os.getcwd(), "../sdk")
+merlin_sdk_package = "merlin-sdk"
+for index, item in enumerate(REQUIRE):
+    if merlin_sdk_package in item:
+        REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}"
 
 setup(
     name='merlin-pyspark-app',
     version='0.2.0',
     author_email='[email protected]',
     description='Base pyspark application for running merlin prediction job',
     long_description=open('README.md').read(),
-    python_requires='>=3.7,<3.11',
+    python_requires='>=3.8,<3.11',
     packages=find_packages("merlinpyspark"),
     install_requires=REQUIRE,
     tests_require=TEST_REQUIRE,

diff --git a/python/batch-predictor/test-model/conda.yaml b/python/batch-predictor/test-model/conda.yaml
@@ -1,4 +1,4 @@
 dependencies:
 - pip:
-  - scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+  - scikit-learn>=1.1.2
   - joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
diff --git a/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/Pipfile b/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/Pipfile
@@ -7,4 +7,4 @@ verify_ssl = true
 merlin-sdk = "=={{ cookiecutter.merlin_sdk_version }}"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"
diff --git a/python/pyfunc-server/docker/Dockerfile b/python/pyfunc-server/docker/Dockerfile
@@ -24,7 +24,7 @@ WORKDIR /pyfunc-server
 RUN if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}; fi
 RUN gsutil cp -r ${MODEL_URL} .
 RUN /bin/bash -c ". activate merlin-model && \
-    sed -i 's/mlflow$/mlflow==1.23.0/' model/conda.yaml && \
+    sed -i 's/mlflow\(\s*==\s*[^ ]*\)\{0,1\}/mlflow==1.26.1/g' model/conda.yaml && \
     conda env update --name merlin-model --file model/conda.yaml && \
     python -m pyfuncserver --model_dir model --dry_run"
 

diff --git a/python/pyfunc-server/docker/env37.yaml b/python/pyfunc-server/docker/env37.yaml
diff --git a/python/pyfunc-server/requirements.txt b/python/pyfunc-server/requirements.txt
@@ -12,5 +12,4 @@ grpcio<1.49.0
 grpcio-reflection<1.49.0
 grpcio-tools<1.49.0
 grpcio-health-checking<1.49.0
-#TODO: Update merlin-sdk dep to: file:${SDK_PATH}#egg=merlin-sdk
-merlin-sdk==0.33.0 # Pin to the version that supports Python 3.7.
+file:${SDK_PATH}#egg=merlin-sdk
diff --git a/python/pyfunc-server/setup.py b/python/pyfunc-server/setup.py
@@ -31,12 +31,11 @@
 # replace merlin relative path in requirements.txt into absolute path
 # setuptools could not install relative path requirements
 
-# TODO: Uncomment below lines after Pyfunc server stops supporting Python 3.7
-# merlin_path = os.path.join(os.getcwd(), "../sdk")
-# merlin_sdk_package = "merlin-sdk"
-# for index, item in enumerate(REQUIRE):
-#     if merlin_sdk_package in item:
-#         REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}"
+merlin_path = os.path.join(os.getcwd(), "../sdk")
+merlin_sdk_package = "merlin-sdk"
+for index, item in enumerate(REQUIRE):
+    if merlin_sdk_package in item:
+        REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}"
 
 setup(
     name='pyfuncserver',
@@ -45,7 +44,7 @@
     description='Model Server implementation for mlflow pyfunc model',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
-    python_requires='>=3.7,<3.11',
+    python_requires='>=3.8,<3.11',
     packages=find_packages(exclude=["test"]),
     install_requires=REQUIRE,
     tests_require=tests_require,

diff --git a/python/sdk/Dockerfile b/python/sdk/Dockerfile
@@ -10,7 +10,5 @@ RUN apt-get update && apt-get install build-essential curl vim wget -y
 
 COPY . .${WORKDIR}
 
-ARG VERSION
-
-RUN pip install merlin-sdk==${VERSION}
-RUN pip install merlin-sdk[test]==${VERSION}
+RUN pip install .
+RUN pip install ".[test]"
diff --git a/python/sdk/setup.py b/python/sdk/setup.py
@@ -30,8 +30,8 @@
     "cookiecutter>=1.7.2",
     "docker>=4.2.1",
     "google-cloud-storage>=1.19.0",
-    "mlflow>=1.2.0,<=1.23.0", #  for py3.11 due to proto -> "mlflow>=1.26.1",
-    "protobuf>=3.0.0,<4.0.0", #  for py3.11 due to proto -> "protobuf>=4.0.0,<5.0dev",
+    "protobuf>=3.0.0,<5.0.0",
+    "mlflow>=1.26.1,<2.0.0",
     "PyPrind>=2.11.2",
     "python_dateutil>=2.5.3",
     "PyYAML>=5.4",
@@ -53,7 +53,7 @@
     "pytest",
     "recursive-diff>=1.0.0",
     "requests",
-    "scikit-learn==1.0.2", #TODO: >=1.1.2 upon python 3.7 deprecation
+    "scikit-learn>=1.1.2",
     "types-python-dateutil",
     "types-PyYAML",
     "types-six",

diff --git a/python/sdk/test/batch/model/env.yaml b/python/sdk/test/batch/model/env.yaml
@@ -1,7 +1,7 @@
 dependencies:
   - pip:
       - mlflow
-      - scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+      - scikit-learn>=1.1.2
       - joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
       - numpy>=1.19.5,<1.23.4 # https://github.com/Azure/MachineLearningNotebooks/issues/1314
       - pandas==1.3.5

diff --git a/python/sdk/test/pyfunc/env.yaml b/python/sdk/test/pyfunc/env.yaml
@@ -3,7 +3,7 @@ dependencies:
   - pip:
     - joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
     - numpy<=1.23.5  # Temporary pin numpy due to https://numpy.org/doc/stable/release/1.20.0-notes.html#numpy-1-20-0-release-notes
-    - scikit-learn==1.0.2  #TODO: >=1.1.2 upon python 3.7 deprecation
+    - scikit-learn>=1.1.2
     - xgboost==1.6.2
     - pytest
     - pytest-xdist==1.34.0