Make KFP dependency optional in build and during runs (#3248)

Make kfp python package an optional dependency, defer processor loading until runtimes are determined, and make elyra.pipeline.kfp.kfp_authentication import optional Signed-off-by: shalberd <[email protected]> Signed-off-by: Luciano Resende <[email protected]> --------- Co-authored-by: Luciano Resende <[email protected]>
elyra-ai · Nov 8, 2024 · 7cc3270 · 7cc3270
1 parent b76377c
commit 7cc3270
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 14 deletions.
diff --git a/docs/source/getting_started/installation.md b/docs/source/getting_started/installation.md
@@ -33,12 +33,15 @@ The instructions below are installing the latest release.
 
 Prior to version 3.1, the `elyra` package included all dependencies. Subsequent releases allow for selective dependency installation:
 
-- `elyra` - install the Elyra core features
+- `elyra` - install the Elyra core features (except kfp dependencies)
 - `elyra[all]` - install core features and all dependencies
+- `elyra[airflow]` - install the Elyra core features and support for [Apache Airflow pipelines](https://github.com/apache/airflow)
+- `elyra[airflow-gitlab]` - install the Elyra core features and GitLab support for [Apache Airflow pipelines](https://github.com/apache/airflow)
+- `elyra[kfp]` - install the Elyra core features and support for [Kubeflow Pipelines](https://github.com/kubeflow/pipelines)
 - `elyra[kfp-tekton]` - install the Elyra core features and support for [Kubeflow Pipelines on Tekton](https://github.com/kubeflow/kfp-tekton)
-- `elyra[gitlab]` - install the Elyra core features and GitLab support for Apache Airflow pipelines
 - `elyra[kfp-examples]` - install the Elyra core features and [Kubeflow Pipelines custom component examples](https://github.com/elyra-ai/examples/tree/main/component-catalog-connectors/kfp-example-components-connector)
 
+
 ### pip
 
 If you use `pip`, install Elyra with:
@@ -94,7 +97,7 @@ conda install -c conda-forge "elyra[all]"
 ```
 
 **NOTE:**
-The Elyra packaging process was changed in version 3.1.0. The [Kubeflow Pipelines on Tekton](https://github.com/kubeflow/kfp-tekton) dependency [is no longer installed by default](https://github.com/elyra-ai/elyra/pull/2043). To install this dependency, you must specify `elyra[all]` or `elyra[kfp-tekton]`.
+The Elyra packaging process was changed in version 4.0. The [Apache Airflow pipelines](https://github.com/apache/airflow) or [Kubeflow Pipelines on Tekton](https://github.com/kubeflow/kfp-tekton) dependencies are no longer installed by default. To install this dependency, you must specify `elyra[all]`, `elyra[kfp]` or `elyra[kfp-tekton]`.
 
 You can also install the Pipeline editor, Code Snippet, Code Viewer, or Script editor extensions individually:
 

diff --git a/elyra/metadata/schemasproviders.py b/elyra/metadata/schemasproviders.py
@@ -26,17 +26,22 @@
 try:
     from kfp_tekton import TektonClient
 except ImportError:
-    # We may not have kfp-tekton available and that's okay!
+    # We may not have kfp-tekton available and that's okay, for example when only using airflow!
     TektonClient = None
 
 from elyra.metadata.schema import SchemasProvider
 from elyra.metadata.schemaspaces import CodeSnippets
 from elyra.metadata.schemaspaces import ComponentCatalogs
 from elyra.metadata.schemaspaces import RuntimeImages
 from elyra.metadata.schemaspaces import Runtimes
-from elyra.pipeline.kfp.kfp_authentication import SupportedAuthProviders
 from elyra.util.gitutil import SupportedGitTypes
 
+try:
+    from elyra.pipeline.kfp.kfp_authentication import SupportedAuthProviders
+except ImportError:
+    # We may not have kfp available and that's okay, for example when only using airflow!
+    SupportedAuthProviders = None
+
 
 class ElyraSchemasProvider(SchemasProvider, metaclass=ABCMeta):
     """Base class used for retrieving Elyra-based schema files from its metadata/schemas directory."""
@@ -107,12 +112,16 @@ def get_schemas(self) -> List[Dict]:
             # For KFP schemas replace placeholders:
             # - properties.metadata.properties.auth_type.enum ({AUTH_PROVIDER_PLACEHOLDERS})
             # - properties.metadata.properties.auth_type.default ({DEFAULT_AUTH_PROVIDER_PLACEHOLDER})
-            auth_type_enum = SupportedAuthProviders.get_provider_names()
-            auth_type_default = SupportedAuthProviders.get_default_provider().name
+            if SupportedAuthProviders is not None:
+                auth_type_enum = SupportedAuthProviders.get_provider_names()
+                auth_type_default = SupportedAuthProviders.get_default_provider().name
 
             for schema in runtime_schemas:
                 if schema["name"] == "kfp":
-                    if schema["properties"]["metadata"]["properties"].get("auth_type") is not None:
+                    if (
+                        schema["properties"]["metadata"]["properties"].get("auth_type") is not None
+                        and SupportedAuthProviders is not None
+                    ):
                         schema["properties"]["metadata"]["properties"]["auth_type"]["enum"] = auth_type_enum
                         schema["properties"]["metadata"]["properties"]["auth_type"]["default"] = auth_type_default
 

diff --git a/elyra/pipeline/registry.py b/elyra/pipeline/registry.py
@@ -57,8 +57,8 @@ def __init__(self, **kwargs):
         for processor in entrypoints.get_group_all("elyra.pipeline.processors"):
             try:
                 # instantiate an actual instance of the processor
-                processor_instance = processor.load()(root_dir=self.root_dir, parent=kwargs.get("parent"))
                 if not self.runtimes or processor.name in self.runtimes:
+                    processor_instance = processor.load()(root_dir=self.root_dir, parent=kwargs.get("parent"))
                     self._add_processor(processor_instance)
                 else:
                     self.log.info(

diff --git a/pyproject.toml b/pyproject.toml
@@ -44,14 +44,13 @@ dependencies = [
     "rfc3986-validator>=0.1.1",
     "tornado>=6.1.0",
     "traitlets>=4.3.2",
-    "typing-extensions>=3.10,<5",  # Cap from kfp
+    "typing-extensions>=3.10",
     "urllib3>=1.26.5",
     "watchdog>=2.1.3",
     "websocket-client",
     "yaspin",
     # see: https://stackoverflow.com/questions/76175487/sudden-importerror-cannot-import-name-appengine-from-requests-packages-urlli
     "appengine-python-standard",
-    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
     "pygithub",
     "black>=22.8.0",
 ]
@@ -88,20 +87,29 @@ test = [
     "requests-unixsocket",
     "kfp-tekton"
 ]
+airflow = [
+
+]
+airflow-gitlab = [
+    "python-gitlab"
+]
+kfp = [
+    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
+    "typing-extensions>=3.10,<5",  # Cap from kfp
+]
 kfp-tekton = [
     "kfp-tekton>=1.5.2"  # requires kfp >= 1.8.19, which contains fix for Jupyterlab
 ]
 kfp-examples = [
     "elyra-examples-kfp-catalog"
 ]
-gitlab = [
-    "python-gitlab"
-]
 # The following is a collection of "non-test" extra dependencies from above.
 all = [
+    "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
     "kfp-tekton>=1.5.2",
     "elyra-examples-kfp-catalog",
     "python-gitlab",
+    "typing-extensions>=3.10,<5",  # Cap from kfp
 ]
 
 # Console scripts