From be3b020446a036ab4a2a0933f38f76842ae4670d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 7 Jul 2023 11:42:44 +0200 Subject: [PATCH] Add section --- .../docs/core/connect-data-platform/spark-setup.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/spark-setup.md b/website/docs/docs/core/connect-data-platform/spark-setup.md index 3852d4bd0d3..6cbb4c9e400 100644 --- a/website/docs/docs/core/connect-data-platform/spark-setup.md +++ b/website/docs/docs/core/connect-data-platform/spark-setup.md @@ -119,9 +119,7 @@ your_profile_name: port: [port] # default 443 user: [user] server_side_parameters: - # cluster configuration parameters, otherwise applied via `SET` statements - # for example: - # "spark.databricks.delta.schema.autoMerge.enabled": True + "spark.driver.memory": "4g" ``` @@ -148,6 +146,8 @@ your_profile_name: auth: [e.g. KERBEROS] kerberos_service_name: [e.g. hive] use_ssl: [true|false] # value of hive.server2.use.SSL, default false + server_side_parameters: + "spark.driver.memory": "4g" ``` @@ -176,6 +176,8 @@ your_profile_name: user: [user] connect_timeout: 60 # default 10 connect_retries: 5 # default 0 + server_side_parameters: + "spark.driver.memory": "4g" ``` @@ -202,7 +204,7 @@ your_profile_name: schema: [database/schema name] host: NA # not used, but required by `dbt-core` server_side_parameters: - "spark.sql.extensions": "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" + "spark.driver.memory": "4g" ``` @@ -231,6 +233,10 @@ connect_retries: 3 +### Server side configuration + +Spark can be customized using [Application Properties](https://spark.apache.org/docs/latest/configuration.html). Using these properties the execution can be customized, for example, to allocate more memory to the driver process. Also, the Spark SQL runtime can be set through these properties. For example, this allows the user to [set a Spark catalogs](https://spark.apache.org/docs/latest/configuration.html#spark-sql). + ## Caveats ### Usage with EMR