fix for DX-87286, added retries (#221)

see comments on internal Jira DX-87286
dremio · Apr 10, 2024 · b341a9f · b341a9f
1 parent 328df7a
commit b341a9f
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 8 deletions.
diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py
@@ -40,6 +40,7 @@
     DremioInternalServerException,
     DremioServiceUnavailableException,
     DremioGatewayTimeoutException,
+    DremioBadRequestException,
 )
 
 from dbt.events import AdapterLogger
@@ -49,7 +50,7 @@
 
 class DremioConnectionManager(SQLConnectionManager):
     TYPE = "dremio"
-    DEFAULT_CONNECTION_RETRIES = 1
+    DEFAULT_CONNECTION_RETRIES = 5
 
     retries = DEFAULT_CONNECTION_RETRIES
 
@@ -220,8 +221,12 @@ def create_catalog(self, relation):
         if database == ("@" + credentials.UID):
             logger.debug("Database is default: creating folders only")
         else:
+            logger.debug(f"Creating space: {database}")
             self._create_space(database, api_parameters)
-        self._create_folders(database, schema, api_parameters)
+
+        if database != credentials.datalake:
+            logger.debug(f"Creating folder(s): {database}.{schema}")
+            self._create_folders(database, schema, api_parameters)
         return
 
     def _make_new_space_json(self, name) -> json:
@@ -248,6 +253,11 @@ def _create_folders(self, database, schema, api_parameters):
                 create_catalog_api(api_parameters, folder_json)
             except DremioAlreadyExistsException:
                 logger.debug(f"Folder {folder} already exists.")
+            except DremioBadRequestException as e:
+               if "Can not create a folder inside a [SOURCE]" in e.message:
+                   logger.debug(f"Ignoring {e}")
+               else:
+                   raise e
 
     def _create_path_list(self, database, schema):
         path = [database]

diff --git a/dbt/adapters/dremio/impl.py b/dbt/adapters/dremio/impl.py
@@ -61,8 +61,7 @@ def convert_time_type(cls, agate_table, col_idx):
         return "time"
 
     def create_schema(self, relation: DremioRelation) -> None:
-        if relation.is_view:
-            self.connections.create_catalog(relation)
+        self.connections.create_catalog(relation)
 
     def drop_schema(self, relation: DremioRelation) -> None:
         if relation.type == "table":

diff --git a/dbt/include/dremio/macros/adapters/metadata.sql b/dbt/include/dremio/macros/adapters/metadata.sql
@@ -37,7 +37,7 @@ limitations under the License.*/
                 else 'no_schema'
             end as table_schema
         ,reflection_name as table_name
-        ,'materializedview' as table_type
+        ,'materialized_view' as table_type
         ,case
             when nullif(external_reflection, '') is not null then 'target: ' || external_reflection
             when arrow_cache then 'arrow cache'
@@ -166,7 +166,7 @@ limitations under the License.*/
           ,replace(case when first_dot < last_dot
           then substr(dataset_name, first_dot + 1, last_dot - first_dot - 1)
           else 'no_schema' end, '"', '') as table_schema
-          ,'materializedview' as table_type
+          ,'materialized_view' as table_type
         from cte1
       )
       select table_catalog, table_name, table_schema, table_type

diff --git a/dbt/include/dremio/macros/materializations/reflection/helpers.sql b/dbt/include/dremio/macros/materializations/reflection/helpers.sql
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.*/
 
 {% macro drop_reflection_if_exists(relation, reflection) %}
-  {% if reflection is not none and reflection.type == 'materializedview' %}
+  {% if reflection is not none and reflection.type == 'materialized_view' %}
     {% call statement('drop reflection') -%}
       alter dataset {{ relation }}
         drop reflection {{ reflection.include(database=False, schema=False) }}

diff --git a/dbt/include/dremio/macros/materializations/reflection/reflection.sql b/dbt/include/dremio/macros/materializations/reflection/reflection.sql
@@ -56,7 +56,7 @@ limitations under the License.*/
 
   {%- set old_relation = adapter.get_relation(database=anchor.database, schema=anchor.schema, identifier=identifier) -%}
   {%- set target_relation = api.Relation.create(
-      identifier=identifier, schema=anchor.schema, database=anchor.database, type='materializedview') -%}
+      identifier=identifier, schema=anchor.schema, database=anchor.database, type='materialized_view') -%}
 
   {%- set reflection_type = dbt_dremio_validate_get_reflection_type(raw_reflection_type) -%}
   {% if (reflection_type == 'raw' and display is none)