From cd86eb2ccd71baccd306d21319cf338eca5fce20 Mon Sep 17 00:00:00 2001 From: Malav Shastri <57682969+malav-shastri@users.noreply.github.com> Date: Fri, 24 Feb 2023 11:31:03 -0500 Subject: [PATCH] Upgrade SKLearn to 1.2.1 (#163) * Upgrade SKLearn to 1.2.1 * Delete buildspecs and refresh pickled models * Correct local integration tests --------- Co-authored-by: Malav Shastri --- README.rst | 4 ++-- docker/{1.0-1 => 1.2-1}/base/Dockerfile.cpu | 2 +- .../{1.0-1 => 1.2-1}/base/Dockerfile_arm.cpu | 0 .../{1.0-1 => 1.2-1}/extension/Dockerfile.cpu | 8 ++++---- docker/{1.0-1 => 1.2-1}/extension/README.md | 12 ++++++------ docker/{1.0-1 => 1.2-1}/final/Dockerfile.cpu | 4 ++-- .../{1.0-1 => 1.2-1}/final/Dockerfile_arm.cpu | 0 .../resources/libffi7_3.3-6_arm64.deb | Bin .../resources/mms/ExecutionParameters.java | 0 .../resources/mms/config.properties.tmp | 0 .../resources/mms/endpoints-1.0.jar | Bin requirements.txt | 2 +- test/conftest.py | 2 +- .../test_multiple_model_endpoint.py | 8 ++++---- .../models/pickled-model-1/sklearn-model | Bin 4877 -> 4252 bytes .../models/pickled-model-2/sklearn-model | Bin 4877 -> 3658 bytes tox.ini | 4 ++-- 17 files changed, 23 insertions(+), 23 deletions(-) rename docker/{1.0-1 => 1.2-1}/base/Dockerfile.cpu (98%) rename docker/{1.0-1 => 1.2-1}/base/Dockerfile_arm.cpu (100%) rename docker/{1.0-1 => 1.2-1}/extension/Dockerfile.cpu (63%) rename docker/{1.0-1 => 1.2-1}/extension/README.md (87%) rename docker/{1.0-1 => 1.2-1}/final/Dockerfile.cpu (96%) rename docker/{1.0-1 => 1.2-1}/final/Dockerfile_arm.cpu (100%) rename docker/{1.0-1 => 1.2-1}/resources/libffi7_3.3-6_arm64.deb (100%) rename docker/{1.0-1 => 1.2-1}/resources/mms/ExecutionParameters.java (100%) rename docker/{1.0-1 => 1.2-1}/resources/mms/config.properties.tmp (100%) rename docker/{1.0-1 => 1.2-1}/resources/mms/endpoints-1.0.jar (100%) diff --git a/README.rst b/README.rst index 553ac01..69f724b 100644 --- a/README.rst +++ b/README.rst @@ -184,7 +184,7 @@ If you want to run local integration tests, then use: # Example pytest test/integration --docker-base-name preprod-sklearn ``\`` - --tag 1.0-1 ``\`` + --tag 1.2-1 ``\`` --py-version 3 ``\`` --framework-version |FRAMEWORK_VERSION| @@ -242,4 +242,4 @@ SageMaker Scikit-learn Container is licensed under the Apache 2.0 License. It is .com, Inc. or its affiliates. All Rights Reserved. The license is available at: http://aws.amazon.com/apache2.0/ -.. |FRAMEWORK_VERSION| replace:: 1.0-1 +.. |FRAMEWORK_VERSION| replace:: 1.2-1 diff --git a/docker/1.0-1/base/Dockerfile.cpu b/docker/1.2-1/base/Dockerfile.cpu similarity index 98% rename from docker/1.0-1/base/Dockerfile.cpu rename to docker/1.2-1/base/Dockerfile.cpu index fab320f..822effc 100644 --- a/docker/1.0-1/base/Dockerfile.cpu +++ b/docker/1.2-1/base/Dockerfile.cpu @@ -116,4 +116,4 @@ ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.U # Install Scikit-Learn # Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4. # Scikit-learn now requires Python 3.6 or newer. -RUN python -m pip install --no-cache -I scikit-learn==1.0.2 +RUN python -m pip install --no-cache -I scikit-learn==1.2.1 diff --git a/docker/1.0-1/base/Dockerfile_arm.cpu b/docker/1.2-1/base/Dockerfile_arm.cpu similarity index 100% rename from docker/1.0-1/base/Dockerfile_arm.cpu rename to docker/1.2-1/base/Dockerfile_arm.cpu diff --git a/docker/1.0-1/extension/Dockerfile.cpu b/docker/1.2-1/extension/Dockerfile.cpu similarity index 63% rename from docker/1.0-1/extension/Dockerfile.cpu rename to docker/1.2-1/extension/Dockerfile.cpu index 823b1a9..1317c8e 100644 --- a/docker/1.0-1/extension/Dockerfile.cpu +++ b/docker/1.2-1/extension/Dockerfile.cpu @@ -1,9 +1,9 @@ -FROM preprod-sklearn:1.0-1 +FROM preprod-sklearn:1.2-1 -RUN pip freeze | grep -q 'scikit-learn==1.0.2'; \ +RUN pip freeze | grep -q 'scikit-learn==1.2.1'; \ if [ $? -eq 0 ]; \ - then echo 'scikit-learn version 1.0.2 requirement met'; \ - else echo 'ERROR: Expected scikit-learn version is 1.0.2, check base images for scikit-learn version' && \ + then echo 'scikit-learn version 1.2.1 requirement met'; \ + else echo 'ERROR: Expected scikit-learn version is 1.2.1, check base images for scikit-learn version' && \ exit 1; fi RUN pip install --upgrade --no-cache --no-deps sagemaker-scikit-learn-extension==2.5.0 diff --git a/docker/1.0-1/extension/README.md b/docker/1.2-1/extension/README.md similarity index 87% rename from docker/1.0-1/extension/README.md rename to docker/1.2-1/extension/README.md index 8bc19b4..b1a0779 100644 --- a/docker/1.0-1/extension/README.md +++ b/docker/1.2-1/extension/README.md @@ -8,15 +8,15 @@ The SageMaker Scikit-learn Extension Container is built in 3 steps. The first 2 The "base" Dockerfile encompass the installation of the framework and all of the dependencies needed. -Tagging scheme is based on --cpu-py. (e.g. 1.0-1-cpu-py3) +Tagging scheme is based on --cpu-py. (e.g. 1.2-1-cpu-py3) All "final" Dockerfiles build images using base images that use the tagging scheme above. ``` -docker build -t sklearn-base:1.0-1-cpu-py3 -f docker/1.0-1/base/Dockerfile.cpu . +docker build -t sklearn-base:1.2-1-cpu-py3 -f docker/1.2-1/base/Dockerfile.cpu . ``` -Notice that this Dockerfile has the updated version of sklearn (1.0.2) installed. +Notice that this Dockerfile has the updated version of sklearn (1.2.1) installed. ### Step 2: Final Image @@ -38,7 +38,7 @@ python setup.py bdist_wheel Then build the final image, like in the sagemaker-sklearn-container ``` -docker build -t preprod-sklearn:1.0-1-cpu-py3 -f docker/1.0-1/final/Dockerfile.cpu . +docker build -t preprod-sklearn:1.2-1-cpu-py3 -f docker/1.2-1/final/Dockerfile.cpu . ``` ### Step 3: Build the extension image for SageMaker Scikit-learn Extension Container @@ -47,10 +47,10 @@ The "extension" Dockerfiles encompass the installation of the SageMaker Autopilo The "extension" Dockerfiles use final images for building. -Build the third additional Dockerfile needed for SageMaker Scikit-learn Extension Container. This Dockerfile specifies a hard dependency on a certain version of scikit-learn (i.e. v1.0.2). +Build the third additional Dockerfile needed for SageMaker Scikit-learn Extension Container. This Dockerfile specifies a hard dependency on a certain version of scikit-learn (i.e. v1.2.1). Tagging scheme is based on extension---cpu-py. (e.g. extension-2.5-1-cpu-py3). Make sure the "extension" image is tagged in accordance with the `extension` (i.e. `extension-2.5-1-cpu-py3`). ``` -docker build -t preprod-sklearn-extension:2.5-1-cpu-py3 -f docker/1.0-1/extension/Dockerfile.cpu . +docker build -t preprod-sklearn-extension:2.5-1-cpu-py3 -f docker/1.2-1/extension/Dockerfile.cpu . ``` \ No newline at end of file diff --git a/docker/1.0-1/final/Dockerfile.cpu b/docker/1.2-1/final/Dockerfile.cpu similarity index 96% rename from docker/1.0-1/final/Dockerfile.cpu rename to docker/1.2-1/final/Dockerfile.cpu index be7d20c..58f1d84 100644 --- a/docker/1.0-1/final/Dockerfile.cpu +++ b/docker/1.2-1/final/Dockerfile.cpu @@ -1,5 +1,5 @@ -FROM sklearn-base:1.0-1 -ENV SAGEMAKER_SKLEARN_VERSION 1.0-1 +FROM sklearn-base:1.2-1 +ENV SAGEMAKER_SKLEARN_VERSION 1.2-1 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true diff --git a/docker/1.0-1/final/Dockerfile_arm.cpu b/docker/1.2-1/final/Dockerfile_arm.cpu similarity index 100% rename from docker/1.0-1/final/Dockerfile_arm.cpu rename to docker/1.2-1/final/Dockerfile_arm.cpu diff --git a/docker/1.0-1/resources/libffi7_3.3-6_arm64.deb b/docker/1.2-1/resources/libffi7_3.3-6_arm64.deb similarity index 100% rename from docker/1.0-1/resources/libffi7_3.3-6_arm64.deb rename to docker/1.2-1/resources/libffi7_3.3-6_arm64.deb diff --git a/docker/1.0-1/resources/mms/ExecutionParameters.java b/docker/1.2-1/resources/mms/ExecutionParameters.java similarity index 100% rename from docker/1.0-1/resources/mms/ExecutionParameters.java rename to docker/1.2-1/resources/mms/ExecutionParameters.java diff --git a/docker/1.0-1/resources/mms/config.properties.tmp b/docker/1.2-1/resources/mms/config.properties.tmp similarity index 100% rename from docker/1.0-1/resources/mms/config.properties.tmp rename to docker/1.2-1/resources/mms/config.properties.tmp diff --git a/docker/1.0-1/resources/mms/endpoints-1.0.jar b/docker/1.2-1/resources/mms/endpoints-1.0.jar similarity index 100% rename from docker/1.0-1/resources/mms/endpoints-1.0.jar rename to docker/1.2-1/resources/mms/endpoints-1.0.jar diff --git a/requirements.txt b/requirements.txt index d2a4751..5d6a9d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ retrying==1.3.3 sagemaker-containers==2.8.6.post2 sagemaker-inference==1.2.0 sagemaker-training==4.0.1 -scikit-learn==1.0.2 +scikit-learn==1.2.1 scipy==1.5.3 six==1.15.0 jinja2==3.0.3 diff --git a/test/conftest.py b/test/conftest.py index 8bda75b..4b46dde 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -43,7 +43,7 @@ def pytest_addoption(parser): parser.addoption('--install-container-support', '-C', action='store_true') parser.addoption('--docker-base-name', default='sk-learn') parser.addoption('--region', default='us-west-2') - parser.addoption('--framework-version', default='1.0.2') + parser.addoption('--framework-version', default='1.2.1') parser.addoption('--py-version', choices=['2', '3'], default=str(sys.version_info.major)) parser.addoption('--processor', choices=['cpu'], default='cpu') # If not specified, will default to {framework-version}-{processor}-py{py-version} diff --git a/test/integration/test_multiple_model_endpoint.py b/test/integration/test_multiple_model_endpoint.py index 64ddb29..f6acf79 100644 --- a/test/integration/test_multiple_model_endpoint.py +++ b/test/integration/test_multiple_model_endpoint.py @@ -150,7 +150,7 @@ def test_load_and_unload_model(): res_json = json.loads(res) assert res_json['status'] == 'Workers scaled' - code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0, 0.0', model_name) + code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0', model_name) assert code == 200, res code, res = make_unload_model_request(model_name) @@ -158,7 +158,7 @@ def test_load_and_unload_model(): res_json = json.loads(res) assert res_json['status'] == "Model \"{}\" unregistered".format(model_name), res - code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0, 0.0', model_name) + code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0', model_name) assert code == 404, res assert res['message'] == "Model not found: {}".format(model_name), res @@ -184,10 +184,10 @@ def test_load_and_unload_two_models(): res_json = json.loads(res) assert res_json['status'] == 'Workers scaled' - code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0, 0.0', model_name_0) + code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0', model_name_0) assert code == 200, res - code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0, 0.0', model_name_1) + code, res = make_invocation_request('0.0, 0.0, 0.0, 0.0, 0.0', model_name_1) assert code == 200, res code, res = make_unload_model_request(model_name_0) diff --git a/test/resources/models/pickled-model-1/sklearn-model b/test/resources/models/pickled-model-1/sklearn-model index 26f7c5a9aac285aabec4c59c4a4fe5771595ff3a..b5bf17704506f5eb168990352fc1fd398357c97c 100644 GIT binary patch literal 4252 zcmdUxU29xL7{_;$Y?Ix*ZPISCtwq|qL}an4ZF5oBNX1AG9wAmQ6dC56oy~Lfyk*Xu z)Zhy$wk9};(h&r`5J3>V6BK$Q)tk}}Q1HHbFMa^^nK_%yCT`o1?8Q28b~10zdFD6& z|Kz+o^x3E7oGG%JyzVo~!x{?{7C1huk!HkMB9yGG(6A8&&zR;4TgBca;_8N)e^0%w z=426Pc>xvJCo9`w1ZNn+^=Xo@1pA+6u9tXGcm?~r#c;VN7{>`pjwfp=<*Y#%=Q`#{ z68oOOQ6+~QlTk6CZ;%Fy1t?oi1zt!J8pJ+Jh?%Oa0a@km6uiBd6jvqAV`x_ zSK#}I8C=dVzYf?%U&eRraXH@FQv5>SOC(0_xa=S@l%4v|O zIT(dWD8m#?!%;W}Gf;slp5BCm$?U9FvV`C`b20aZl0_$qL?Sqi)ut>)kwX#}&slBB zBO!S$auRjSNS)E3Oza#T}l9fGnkl`)vKp==Gnlr;#` zZ5D8Jx&CLy8jjg8vola_|0$|s%7U4wQNqo-Xb6znVlM`WEFeX{6AWI>MQW2*jAzFB)jst+A z9BrQ-4r62($*g3!jl;yg*i|n7@&1n!od{=b*r5nZRso*EcaL`p8nY?Lf;J~Io!)I$ z-HR5}4n-=sdLACv=%BU@E{~0~a=_b#>*5|rQtUYW$ab2a>B8s(aa4u>Wc9y?ejTYrwoA(w!zPC8DG_-w@nYy~9@yjFY)KBs5 z>cSY`fQJJqp4j)_=`wAWqz)X16Zm^DWkwf8?V-wqWz^WSVBJq?D-Ti;ylmy+l`b1X ziK6@-$_q~3UORdFm-=8AUhqzrxc>q5S6#C`B&ct{Sn&HBlrW&?mkt8zMN_7Fp{QT) z{Qg{jgL*^@YVA~42kuW%rCxyg;j3H2{SC@+`ttNr;Q-dl6jkg6D7${4-rt}y>x~}5 zdOZOpzxk=pt#`%9;>3Xzm5Hz?|MhOJ|MtI7)ae7=QJM9Q_riKVfBn&He_QWCpHut0 f%zL7!&p){PW`BeF#C&%?dI&}B{ybpWcW3Qi3bUD^ literal 4877 zcmbuBd3@Z&701_c;yCL#wsVjW2niv94G`m;7)UUXkU-RK+9?GA397V~GaGhS`y0(B zaf@0gjUCgA-Y0G8N$>l@|OTMcr{eC6%%U;RvcSZu`hdL|V%5YHLWnP4TfP7(~vdFBv0c{axHut1-og~T=smr@mwRmt7`-WJ$XZd zeH`mlf?@LZ=1rtB5S>uF+DBjBUc$eo#ERu=jg~t~DHoJa5V0^lg(2tElfj~V9Z;d_ zxTUIev3M$&7v#i6bPoB7F=L56hujnm)&xK?2)~4q3n~3HI6qKctM~;t=uY>A2W_1@SLzfJ%SS^><*f<;v5XT5GU?= zm@Y{xo=Z+j%L{0YE3Ay2rT4?GDl-$MangjO)pe1R0mk*h$~7uKYJDPQrUkEC+ zv|2%DvAQ0Wtqj*kaf+2WiZwghJggnZsRF0vtTwCTC~|S&upX`YS*){X(X4w1ar(ru zahxG=X3lE2+sJnv*5~ti1H~9?NTc7I*pIVvrc?t|stp)uJa5E?#`7j@Y&>tqrpEIY zY;HVn#TH*K9HNK=!(J^Svv)Jv9TQ9e-}NjXVQg7(my+i+>%09ilbJl0GZ6xhi!-^5 zGHIu<%eQRB<2jYf1+Jh}kXr~~HrD50xy)ws7-E*bcJ#iP~DzoFXq(esr z6o0ij5hqU)?2Bt}lAPQ~`|YJPd;J)pqqJg9r^YtBM#G8lgq*aefVrHdCs5!lABk$% zvod4w<(0Bvb&g>~UPTYRV>lqMjx|OGAz7m+aBzB!Ydve(7)o-VJZP;NLs{-8jocV2 z@*46Cj3KC5)Eri?MwKcv$VW-?dMJLOmGO1Ev1BGISbu+T2+&ZTIhGE&r5c^e;bfgO z*Qq9O$kd6uW*mp*L283ZF&xnt^=h76BIyJ*{r|)WPingVZ`#=%{?GoG)a`OeCR+JA zPAn35a@;^qk-L+APc7U7LegL`1boqJ~h z?oFK7n+4tyFZ-=&aBtIiy8-tOfpSk%hqw!2OmJdr;ta@v?uP2KNVzKN@g<68Lk3zf1$}udMJlfxpKJ z|7Z=|KQ;c90QV59KP(jg6Hl>IZS(~& zZP6}F`K{i4EOqQkj*3(_6ezFPrXYpW){s4Kj+CVl@-eHRS*Y$xTvydN`nuG7!?uJMT-;~kU(fOyB_B)cz+hvq{4d#W+wYsc9IzDHiP_nvBy|(YZ6c4xAF3yHNSGU#KkJSgN zE(>VoxK!YrtoGap&d`P3rD4cIoPUwoPU!gF6`UXD!{&})91WCQ2=^k&S(`A4~TixviRv|Gs{g5%7_*xO1LEZ-NQ;51MNvf%p`32i)Qbtq>& z@{VtX>hj;%plZrFjLY&dF(V_3*C=;204~%ateJ9NQ%Vm(nHWk5mP{xcQ&2IcVY$x& zPHk=ep0I`$6IPEFYW=UWlHw;Gb@5D*E(*wt++e?<3J5nR>OU4iA#mX_>vQOw)|h zjz5e|p{&EVQRvxqC9@asc?O@0_-rVKYjV-+3nmWM$jPEd{73|m2#FS6ao_-;Am{t1 zhs)?0@<}Ph+!kQ*f!I|q-|^n}2NABBa6}PWMh;%UcP|VI8l%a{oGwmex_rD?^(>Z{ zu27_cyU)TojSd<;a9JE&kWOYGz_Vsu7Q3t?hz(8`6+F-5a!yBcsdP};0+EEMI=I`s zb+`G*&wi4<#$+q%H!Y}JwUHE+fLa>| ysBbsEe`2yhr9v7BRBx~Tq5Pk}dJli{`74tRD!I&0w?-+w1k~~|MX}^xLgOEpEz=VK literal 4877 zcmbuBd3@Z&701_c;yCL#wsVjW2niv94G`m;7)UUXkU-RK+9?GA397V~GaGhS`y0(B zaf@0gjUCgA-Y0G8N$>l@|OTMcr{eC6%%U;RvcSZu`hdL|V%5YHLWnP4TfP7(~vdFBv0c{axHut1-og~T=smr@mwRmt7`-WJ$XZd zeH`mlf?@LZ=1rtB5S>uF+DBjBUc$eo#ERu=jg~t~DHoJa5V0^lg(2tElfj~V9Z;d_ zxTUIev3M$&7v#i6bPoB7F=L56hujnm)&xK?2)~4q3n~3HI6qKctM~;t=uY>A2W_1@SLzfJ%SS^><*f<;v5XT5GU?= zm@Y{xo=Z+j%L{0YE3Ay2rT4?GDl-$MangjO)pe1R0mk*h$~7uKYJDPQrUkEC+ zv|2%DvAQ0Wtqj*kaf+2WiZwghJggnZsRF0vtTwCTC~|S&upX`YS*){X(X4w1ar(ru zahxG=X3lE2+sJnv*5~ti1H~9?NTc7I*pIVvrc?t|stp)uJa5E?#`7j@Y&>tqrpEIY zY;HVn#TH*K9HNK=!(J^Svv)Jv9TQ9e-}NjXVQg7(my+i+>%09ilbJl0GZ6xhi!-^5 zGHIu<%eQRB<2jYf1+Jh}kXr~~HrD50xy)ws7-E*bcJ#iP~DzoFXq(esr z6o0ij5hqU)?2Bt}lAPQ~`|YJPd;J)pqqJg9r^YtBM#G8lgq*aefVrHdCs5!lABk$% zvod4w<(0Bvb&g>~UPTYRV>lqMjx|OGAz7m+aBzB!Ydve(7)o-VJZP;NLs{-8jocV2 z@*46Cj3KC5)Eri?MwKcv$VW-?dMJLOmGO1Ev1BGISbu+T2+&ZTIhGE&r5c^e;bfgO z*Qq9O$kd6uW*mp*L283ZF&xnt^=h76BIyJ*{r|)WPingVZ`#=%{?GoG)a`OeCR+JA zPAn35a@;^qk-L+APc7U7LegL`1boqJ~h z?oFK7n+4tyFZ-=&aBtIiy8-tOfpSk%hqw!2OmJdr;ta@v?uP2KNVzKN@g<68Lk3zf1$}udMJlfxpKJ z|7Z=|KQ;c90QV59KP(jg6Hl>IZS(~& zZP6}F`K{i4EOqQkj*3(_6ezFPrXYpW){s4Kj+CVl