feat: remove databricks

UiPath · Jul 10, 2024 · 3e50527 · 3e50527
1 parent 19309ae
commit 3e50527
Show file tree

Hide file tree

Showing 37 changed files with 308 additions and 652 deletions.
diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml
@@ -12,7 +12,7 @@ resources:
       endpoint: UiPath
       type: github
       name: UiPath/ProcessMining-framework-resources
-      ref: refs/tags/test_transformations.1.2.1
+      ref: main
 
 # Trigger the pipeline for all PRs we open, but do not start as long as there is no PR.
 trigger:
@@ -40,10 +40,6 @@ jobs:
     dependsOn: get_keyvault_secrets
     condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded')
     variables:
-        DBT_DATABRICKS_CATALOG: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_DATABRICKS_CATALOG'] ]
-        DBT_DATABRICKS_HOST: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_DATABRICKS_HOST'] ]
-        DBT_DATABRICKS_HTTP_PATH: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_DATABRICKS_HTTP_PATH'] ]
-        DBT_DATABRICKS_TOKEN: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_DATABRICKS_TOKEN'] ]
         DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ]
         DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ]
         DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ]
@@ -66,16 +62,8 @@ jobs:
       - bash: |
           echo "##vso[task.setvariable variable=DBT_SCHEMA]pm_utils_integration_tests"
           echo "##vso[task.setvariable variable=dbtProjectPath]$(Agent.BuildDirectory)/self/integration_tests"
-          echo "##vso[task.setvariable variable=dbtProjectPathQuotes]$(Agent.BuildDirectory)/self/transformations_double_quotes"
         displayName: Set variables
 
-      # Replace quotes in the dbt project to run it on SQL Server and Snowflake. The replaced version of the project is stored as "transformations_double_quotes".
-      - task: PowerShell@2
-        inputs:
-          filePath: $(Agent.BuildDirectory)/framework_resources/.pipelines/resources/copy_and_replace_quotes.ps1
-          arguments: -folderPath "$(dbtProjectPath)"
-        displayName: "Create dbt project with double quotes" 
-
       - bash: |
           python --version
 
@@ -87,7 +75,6 @@ jobs:
           python3 -m venv dbt-env-sqlserver
 
           source dbt-env/bin/activate
-          pip install dbt-databricks==1.7.7
           pip install dbt-snowflake==1.7.5
           dbt --version
 
@@ -102,29 +89,16 @@ jobs:
           dbt deps
         displayName: Install dbt dependencies
 
-      - bash: |
-          source dbt-env/bin/activate
-          cd $(dbtProjectPath)
-          dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t databricks-ci \
-            --vars '{"DBT_DATABRICKS_CATALOG": "$(DBT_DATABRICKS_CATALOG)", "DBT_DATABRICKS_HOST": "$(DBT_DATABRICKS_HOST)", "DBT_DATABRICKS_HTTP_PATH": "$(DBT_DATABRICKS_HTTP_PATH)", "DBT_DATABRICKS_TOKEN": "$(DBT_DATABRICKS_TOKEN)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
-        displayName: Test (Databricks)
-
-      - bash: |
-          source dbt-env/bin/activate
-          cd $(dbtProjectPathQuotes)
-          dbt deps
-        displayName: Install dbt dependencies (double quotes) 
-
       - bash: |
           source dbt-env-sqlserver/bin/activate
-          cd $(dbtProjectPathQuotes)
+          cd $(dbtProjectPath)
           dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t sqlserver-ci \
             --vars '{"DBT_SQL_SERVER_SERVER": "$(DBT_SQL_SERVER_SERVER)", "DBT_SQL_SERVER_USER": "$(DBT_SQL_SERVER_USER)", "DBT_SQL_SERVER_PASSWORD": "$(DBT_SQL_SERVER_PASSWORD)", "DBT_SQL_SERVER_DATABASE": "$(DBT_SQL_SERVER_DATABASE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
         displayName: Test (SQL Server)
 
       - bash: |
           source dbt-env/bin/activate
-          cd $(dbtProjectPathQuotes)
+          cd $(dbtProjectPath)
           dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t snowflake-ci \
             --vars '{"DBT_SNOWFLAKE_ACCOUNT": "$(DBT_SNOWFLAKE_ACCOUNT)", "DBT_SNOWFLAKE_USER": "$(DBT_SNOWFLAKE_USER)", "DBT_SNOWFLAKE_PASSWORD": "$(DBT_SNOWFLAKE_PASSWORD)", "DBT_SNOWFLAKE_ROLE": "$(DBT_SNOWFLAKE_ROLE)", "DBT_SNOWFLAKE_DATABASE": "$(DBT_SNOWFLAKE_DATABASE)", "DBT_SNOWFLAKE_WAREHOUSE": "$(DBT_SNOWFLAKE_WAREHOUSE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
         displayName: Test (Snowflake)
diff --git a/.pipelines/profiles.yml b/.pipelines/profiles.yml
@@ -24,15 +24,4 @@ default:
       schema: "{{ var('DBT_SCHEMA') }}"
       threads: 10
 
-    databricks-ci:
-      type: databricks
-      catalog: "{{ var('DBT_DATABRICKS_CATALOG') }}"
-      schema: "{{ var('DBT_SCHEMA') }}"
-      host: "{{ var('DBT_DATABRICKS_HOST') }}"
-      http_path: "{{ var('DBT_DATABRICKS_HTTP_PATH') }}"
-      token: "{{ var('DBT_DATABRICKS_TOKEN') }}"
-      threads: 10
-      session_properties:
-        legacy_time_parser_policy: LEGACY
-
-  target: databricks-ci
+  target: snowflake-ci
diff --git a/README.md b/README.md
@@ -15,14 +15,14 @@ This package contains some date/datetime conversion macros. You can override the
 ```
 vars:
   # Date and time formats.
-  # For SQL Server defined by integers and for Snowflake/Databricks defined by strings.
-  date_format: 23     # default: SQL Server: 23, Snowflake: 'YYYY-MM-DD', Databricks: 'yyyy-MM-dd'
-  time_format: 14     # default: SQL Server: 14, Snowflake: 'hh24:mi:ss.ff3', Databricks: 'HH:mm:ss'
-  datetime_format: 21 # default: SQL Server: 21, Snowflake: 'YYYY-MM-DD hh24:mi:ss.ff3', Databricks: 'yyyy-MM-dd HH:mm:ss'
+  # For SQL Server defined by integers and for Snowflake defined by strings.
+  date_format: 23     # default: SQL Server: 23, Snowflake: 'YYYY-MM-DD'
+  time_format: 14     # default: SQL Server: 14, Snowflake: 'hh24:mi:ss.ff3'
+  datetime_format: 21 # default: SQL Server: 21, Snowflake: 'YYYY-MM-DD hh24:mi:ss.ff3'
 ```
 
 ## Contents
-This dbt package contains macros for SQL functions to run the dbt project on multiple databases. The databases that are currently supported are Snowflake, SQL Server and Databricks.
+This dbt package contains macros for SQL functions to run the dbt project on multiple databases. The databases that are currently supported are Snowflake and SQL Server.
 
 - [SQL generators](#SQL-generators)
   - [create_index](#create_index-source)
@@ -203,7 +203,7 @@ Usage:
 `{{ pm_utils.to_integer('[expression]') }}`
 
 #### to_timestamp ([source](macros/data_type_cast_functions/to_timestamp.sql))
-This macro converts a field to a timestamp field. For Databricks, milliseconds will automatically be parsed. You should only set the milliseconds in your datetime patern if the second fraction is different than milliseconds.
+This macro converts a field to a timestamp field. 
 
 Usage: 
 `{{ pm_utils.to_timestamp('[expression]') }}`

diff --git a/integration_tests/models/input_table.sql b/integration_tests/models/input_table.sql
@@ -1,4 +1,4 @@
 {# This creates a table that can be used in tests that require a relation as argument. #}
 select
-    'A' as `Column_A`,
-    'B' as `Column_B`
+    'A' as "Column_A",
+    'B' as "Column_B"