Support to pass creds for custom docker registry, fix for cloudpickle version incompatibility, python logging api incompatibility

clipper_admin/clipper_admin/clipper_admin.py

@@ -348,7 +348,7 @@ def build_and_deploy_model(self,
             raise UnconnectedException()
         image = self.build_model(name, version, model_data_path, base_image,
                                  container_registry, pkgs_to_install)
-        self.deploy_model(name, version, input_type, image, labels,
+        self.deploy_model(name, version, input_type, container_registry, image, labels,
                           num_replicas, batch_size)
 
     def build_model(self,
@@ -389,7 +389,7 @@ def build_model(self,
             container RPC client.
         container_registry : str, optional
             The Docker container registry to push the freshly built model to. Note
-            that if you are running Clipper on Kubernetes, this registry must be accesible
+            that if you are running Clipper on Kubernetes, this registry must be accessible
             to the Kubernetes cluster in order to fetch the container from the registry.
         pkgs_to_install : list (of strings), optional
             A list of the names of packages to install, using pip, in the container.
@@ -480,6 +480,7 @@ def deploy_model(self,
                      name,
                      version,
                      input_type,
+                     container_registry,
                      image,
                      labels=None,
                      num_replicas=1,
@@ -515,6 +516,11 @@ def deploy_model(self,
             one of "integers", "floats", "doubles", "bytes", or "strings". See the
             `User Guide <http://clipper.ai/user_guide/#input-types>`_ for more details
             on picking the right input type for your application.
+        container_registry : str, optional
+            The Docker container registry to push the freshly built model to. Note
+            that if you are running Clipper on Kubernetes, you should create kubernetes secret with the
+            credentials needed to access the registry, and pass the secret name using `registry_secret_names`
+             parameter in the KubernetesContainerManager constructor.
         image : str
              The fully specified Docker image to deploy. If using a custom
              registry, the registry name must be prepended to the image. For example,
@@ -556,6 +562,7 @@ def deploy_model(self,
             name=name,
             version=version,
             input_type=input_type,
+            container_registry=container_registry,
             image=image,
             num_replicas=num_replicas)
         self.register_model(

clipper_admin/clipper_admin/deployers/pyspark.py

@@ -222,7 +222,7 @@ def predict(spark, model, inputs):
         else:
             pyspark_model.save(sc, spark_model_save_loc)
     except Exception as e:
-        logger.warn("Error saving spark model: %s" % e)
+        logger.warning("Error saving spark model: %s" % e)
         raise e
 
     # extract the pyspark class name. This will be something like

clipper_admin/clipper_admin/deployers/tensorflow.py

@@ -191,7 +191,7 @@ def predict(sess, inputs):
             save_path = saver.save(tf_sess_or_saved_model_path,
                                    tf_sess_save_loc)
         except Exception as e:
-            logger.warn("Error saving Tensorflow model: %s" % e)
+            logger.warning("Error saving Tensorflow model: %s" % e)
             raise e
         logger.info("TensorFlow model saved at: %s " % save_path)
     else:

clipper_admin/clipper_admin/docker/docker_container_manager.py

@@ -268,11 +268,11 @@ def connect(self):
         self.prometheus_port = all_labels[CLIPPER_DOCKER_PORT_LABELS['metric']]
         self.prom_config_path = all_labels[CLIPPER_METRIC_CONFIG_LABEL]
 
-    def deploy_model(self, name, version, input_type, image, num_replicas=1):
+    def deploy_model(self, name, version, input_type, image, container_registry, num_replicas=1):
         # Parameters
         # ----------
         # image : str
-        #     The fully specified Docker imagesitory to deploy. If using a custom
+        #     The fully specified Docker image to deploy. If using a custom
         #     registry, the registry name must be prepended to the image. For example,
         #     "localhost:5000/my_model_name:my_model_version" or
         #     "quay.io/my_namespace/my_model_name:my_model_version"

clipper_admin/clipper_admin/kubernetes/kubernetes_container_manager.py

@@ -68,7 +68,8 @@ def __init__(self,
                  redis_port=6379,
                  useInternalIP=False,
                  namespace='default',
-                 create_namespace_if_not_exists=False):
+                 create_namespace_if_not_exists=False,
+                 registry_secret_names={}):
         """
 
         Parameters
@@ -105,6 +106,10 @@ def __init__(self,
             Create a k8s namespace if the namespace doesnt already exist.
             If this argument is provided and the k8s namespace does not exist a new k8s namespace will
             be created.
+         registry_secret_names: Dictionary, {}
+            A mapping between the container registry where the images reside and the name of the
+            secret that contains the credentials to access the registry. E.g.
+                registry_secret_names = {"localhost:5000": "myregistrykey"}
 
         Note
         ----
@@ -129,6 +134,7 @@ def __init__(self,
         configuration.assert_hostname = False
         self._k8s_v1 = client.CoreV1Api()
         self._k8s_beta = client.ExtensionsV1beta1Api()
+        self.registry_secret_names = registry_secret_names
 
         # Create the template engine
         # Config: Any variable missing -> Error
@@ -301,7 +307,7 @@ def connect(self):
 
         if len(external_node_hosts) == 0 and (self.useInternalIP):
             msg = "No external node addresses found. Using Internal IP address"
-            self.logger.warn(msg)
+            self.logger.warning(msg)
             for addr in node.status.addresses:
                 if addr.type == "InternalIP":
                     external_node_hosts.append(addr.address)
@@ -361,18 +367,19 @@ def connect(self):
                             self.clipper_metric_port))
 
         except ApiException as e:
-            logging.warn(
+            logging.warning(
                 "Exception connecting to Clipper Kubernetes cluster: {}".
                 format(e))
             raise ClipperException(
                 "Could not connect to Clipper Kubernetes cluster. "
                 "Reason: {}".format(e))
 
-    def deploy_model(self, name, version, input_type, image, num_replicas=1):
+    def deploy_model(self, name, version, input_type, image, container_registry, num_replicas=1):
         for query_frontend_id in range(self.num_frontend_replicas):
             deployment_name = get_model_deployment_name(
                 name, version, query_frontend_id, self.cluster_name)
 
+            secret_name = self.registry_secret_names.get(container_registry, "")
             generated_body = self._generate_config(
                 CONFIG_FILES['model']['deployment'],
                 deployment_name=deployment_name,
@@ -383,7 +390,8 @@ def deploy_model(self, name, version, input_type, image, num_replicas=1):
                 query_frontend_id=query_frontend_id,
                 input_type=input_type,
                 image=image,
-                cluster_name=self.cluster_name)
+                cluster_name=self.cluster_name,
+                image_secret_name=secret_name)
 
             with _pass_conflicts():
                 self._k8s_beta.create_namespaced_deployment(
@@ -462,7 +470,7 @@ def stop_models(self, models):
                             cluster_label=CLIPPER_DOCKER_LABEL,
                             cluster_name=self.cluster_name))
         except ApiException as e:
-            self.logger.warn(
+            self.logger.warning(
                 "Exception deleting kubernetes deployments: {}".format(e))
             raise e
 
@@ -476,7 +484,7 @@ def stop_all_model_containers(self):
                     cluster_label=CLIPPER_DOCKER_LABEL,
                     cluster_name=self.cluster_name))
         except ApiException as e:
-            self.logger.warn(
+            self.logger.warning(
                 "Exception deleting kubernetes deployments: {}".format(e))
             raise e
 
@@ -511,7 +519,7 @@ def stop_all(self, graceful=True):
             self._k8s_v1.delete_collection_namespaced_config_map(
                 namespace=self.k8s_namespace, label_selector=cluster_selecter)
         except ApiException as e:
-            logging.warn(
+            logging.warning(
                 "Exception deleting kubernetes resources: {}".format(e))
 
     def get_registry(self):

clipper_admin/clipper_admin/kubernetes/model-container-template.yaml

@@ -43,3 +43,7 @@ spec:
                 - /model_is_ready.check
             initialDelaySeconds: 3
             periodSeconds: 3
+      {% if image_secret_name %}
+      imagePullSecrets:
+        - name: {{ image_secret_name }}
+      {% endif %}

clipper_admin/clipper_admin/kubernetes/query-frontend-deployment.yaml

@@ -38,4 +38,3 @@ spec:
         ports:
         - containerPort: 1390
       restartPolicy: Always
-

clipper_admin/setup.py

@@ -25,7 +25,7 @@
     install_requires=[
         'requests', 'numpy', 'subprocess32; python_version<"3"', 'pyyaml',
         'docker>=3.0', 'kubernetes>=6.0.0', 'prometheus_client',
-        'cloudpickle>=0.5', 'enum34; python_version<"3.4"', 'redis', 'psutil',
+        'cloudpickle==0.5.5', 'enum34; python_version<"3.4"', 'redis', 'psutil',
         'jsonschema', 'jinja2'
     ],
     extras_require={

dockerfiles/ClipperDevDockerfile

@@ -66,7 +66,7 @@ RUN set -ex; \
 # see CA_CERTIFICATES_JAVA_VERSION notes above
 RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
 
-RUN pip install cloudpickle==0.5.* pyzmq==17.0.* requests==2.18.* subprocess32==3.2.* scikit-learn==0.19.* \
+RUN pip install cloudpickle==0.5.5 pyzmq==17.0.* requests==2.18.* subprocess32==3.2.* scikit-learn==0.19.* \
   numpy==1.14.* pyyaml==3.12.* docker==3.1.* kubernetes==6.0.* tensorflow==1.6.* mxnet==1.1.* pyspark==2.3.* \
   xgboost==0.7.* jsonschema==2.6.* psutil==5.4.* prometheus_client
 

dockerfiles/ClipperPy35DevDockerfile

@@ -71,7 +71,7 @@ RUN set -ex; \
 # see CA_CERTIFICATES_JAVA_VERSION notes above
 RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
 
-RUN pip3 install cloudpickle==0.5.* pyzmq==17.0.* requests==2.18.* scikit-learn==0.19.* \
+RUN pip3 install cloudpickle==0.5.5 pyzmq==17.0.* requests==2.18.* scikit-learn==0.19.* \
   numpy==1.14.* pyyaml==3.12.* docker==3.1.* kubernetes==5.0.* tensorflow==1.6.* mxnet==1.1.* pyspark==2.3.* \
   xgboost==0.7.*
 

dockerfiles/Py2RPCDockerfile

@@ -9,7 +9,7 @@ RUN mkdir -p /model \
       && apt-get update \
       && apt-get install -y libzmq5 libzmq5-dev redis-server libsodium18 build-essential
 
-RUN pip install cloudpickle==0.5.* pyzmq==17.0.* prometheus_client==0.1.* \
+RUN pip install cloudpickle==0.5.5 pyzmq==17.0.* prometheus_client==0.1.* \
     pyyaml==3.12.* jsonschema==2.6.* redis==2.10.* psutil==5.4.* flask==0.12.2 \
     numpy==1.14.* subprocess32==3.2.*
 

dockerfiles/Py35RPCDockerfile

@@ -9,7 +9,7 @@ RUN mkdir -p /model \
       && apt-get update \
       && apt-get install -y libzmq3 libzmq3-dev redis-server libsodium13 build-essential
 
-RUN pip install cloudpickle==0.5.* pyzmq==17.0.* prometheus_client==0.1.* \
+RUN pip install cloudpickle==0.5.5 pyzmq==17.0.* prometheus_client==0.1.* \
     pyyaml==3.12.* jsonschema==2.6.* redis==2.10.* psutil==5.4.* flask==0.12.2 \
     numpy==1.14.*
 

dockerfiles/Py36RPCDockerfile

@@ -9,7 +9,7 @@ RUN mkdir -p /model \
       && apt-get update \
       && apt-get install -y libzmq5 libzmq5-dev redis-server libsodium18 build-essential
 
-RUN pip install cloudpickle==0.5.* pyzmq==17.0.* prometheus_client==0.1.* \
+RUN pip install cloudpickle==0.5.5 pyzmq==17.0.* prometheus_client==0.1.* \
     pyyaml==3.12.* jsonschema==2.6.* redis==2.10.* psutil==5.4.* flask==0.12.2 \
     numpy==1.14.*
 

integration-tests/clipper_admin_tests.py

@@ -274,6 +274,7 @@ def test_stop_models(self):
                     model_name,
                     version,
                     input_type,
+                    clipper_registry,
                     container_name,
                     num_replicas=1)