From d3876eeed7bb14df1a8ea6613132206af55a16b1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Sat, 21 Sep 2024 14:19:08 +0800 Subject: [PATCH 1/3] use V2 instead Signed-off-by: Future-Outlier --- .../databricks_agent/databricks_agent_example_usage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py index 9de652bc3..715a2db61 100644 --- a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py +++ b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py @@ -11,7 +11,7 @@ import flytekit from flytekit import Resources, task, workflow -from flytekitplugins.spark import Databricks +from flytekitplugins.spark import DatabricksV2 # %% [markdown] @@ -20,7 +20,7 @@ # [Databricks job request](https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure) documentation. # %% @task( - task_config=Databricks( + task_config=DatabricksV2( spark_conf={ "spark.driver.memory": "1000M", "spark.executor.memory": "1000M", From c0beecd7f0691c430b141bf95186e42821f07353 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 24 Sep 2024 09:55:39 +0800 Subject: [PATCH 2/3] Update examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier --- .../databricks_agent/databricks_agent_example_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py index 715a2db61..b2d04a86a 100644 --- a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py +++ b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py @@ -20,7 +20,7 @@ # [Databricks job request](https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure) documentation. # %% @task( - task_config=DatabricksV2( + task_config=Databricks( spark_conf={ "spark.driver.memory": "1000M", "spark.executor.memory": "1000M", From 71fc982db4724228a7ff58f691c8600771957e8d Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 24 Sep 2024 09:55:45 +0800 Subject: [PATCH 3/3] Update examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier --- .../databricks_agent/databricks_agent_example_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py index b2d04a86a..1fe9d3ff9 100644 --- a/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py +++ b/examples/databricks_agent/databricks_agent/databricks_agent_example_usage.py @@ -11,7 +11,7 @@ import flytekit from flytekit import Resources, task, workflow -from flytekitplugins.spark import DatabricksV2 +from flytekitplugins.spark import DatabricksV2 as Databricks # %% [markdown]