SPIKE: external tables as model materialization

dbt/adapters/base/relation.py

@@ -351,6 +351,14 @@ def is_view(self) -> bool:
     def is_materialized_view(self) -> bool:
         return self.type == RelationType.MaterializedView
 
+    @property
+    def is_external(self) -> bool:
+        return self.type == RelationType.External
+
+    @property
+    def is_external_table(self) -> bool:
+        return self.type == RelationType.ExternalTable
+
     @classproperty
     def Table(cls) -> str:
         return str(RelationType.Table)
@@ -367,6 +375,10 @@ def View(cls) -> str:
     def External(cls) -> str:
         return str(RelationType.External)
 
+    @classproperty
+    def ExternalTable(cls) -> str:
+        return str(RelationType.ExternalTable)
+
     @classproperty
     def MaterializedView(cls) -> str:
         return str(RelationType.MaterializedView)

dbt/adapters/contracts/relation.py

@@ -20,6 +20,7 @@ class RelationType(StrEnum):
     MaterializedView = "materialized_view"
     External = "external"
     Ephemeral = "ephemeral"
+    ExternalTable = "external_table"
 
 
 class MaterializationContract(Protocol):

dbt/include/global_project/macros/materializations/external_table/external_table.sql

@@ -0,0 +1,58 @@
+{% materialization external_table, default %}
+
+  {%- set identifier = model['alias'] -%}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+
+  {%- set existing_relation = load_cached_relation(this) -%}
+  {%- set target_relation = this.incorporate(type=this.ExternalTable) %}
+
+  {{ log('existing_relation: ' ~ existing_relation, info=True) }}
+  {{ log('existing_relation.type: ' ~ existing_relation.type, info=True) }}
+
+  {%- set exists_as_table = (existing_relation is not none and existing_relation.is_table) -%}
+  {%- set exists_as_view = (existing_relation is not none and existing_relation.is_view) -%}
+  {%- set exists_as_external_table = (existing_relation is not none and existing_relation.is_external_table) -%}
+
+  -- build model
+  {% set build_plan = [] %}
+  {% set code = 'CREATE' %}
+
+  {% if exists_as_view %}
+      {% set build_plan = build_plan + [
+      drop_relation_if_exists(existing_relation),
+      create_external_table(target_relation, model.columns.values())
+    ] %}
+   {% elif exists_as_table %}
+    {% if full_refresh_mode %}
+      {% set build_plan = build_plan + [create_external_table(target_relation, model.columns.values())] %}
+    {% elif not full_refresh_mode %} 
+      {% set code = 'REFRESH' %}
+      {% set build_plan = build_plan + refresh_external_table(target_relation) %}
+    {% endif %}
+  {% else %}
+    {% set build_plan = build_plan + [
+      create_external_table(target_relation, model.columns.values())
+    ] %}
+  {% endif %}
+
+  {% call noop_statement('main', code, code) %}
+    {{ build_plan }};
+  {% endcall %}
+
+  {%- set grant_config = config.get('grants') -%}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}

dbt/include/global_project/macros/materializations/external_table/get_create_external_table.sql

@@ -0,0 +1,7 @@
+{% macro create_external_table(relation, columns) %}
+    {{ adapter.dispatch('create_external_table', 'dbt')(relation, columns) }}
+{% endmacro %}
+
+{% macro default__create_external_table(relation, columns) %}
+    {{ exceptions.raise_compiler_error("External table creation is not implemented for the default adapter") }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/external_table/helpers.sql

@@ -0,0 +1,54 @@
+{% macro refresh_external_table(source_node) %}
+    {{ return(adapter.dispatch('refresh_external_table', 'dbt')(source_node)) }}
+{% endmacro %}
+
+{% macro default__refresh_external_table(source_node) %}
+    {% do return([]) %}
+{% endmacro %}
+
+{% macro update_external_table_columns(source_node) %}
+    {{ return(adapter.dispatch('update_external_table_columns', 'dbt')(source_node)) }}
+{% endmacro %}
+
+{% macro default__update_external_table_columns(source_node) %}
+
+{% endmacro %}
+
+{%- macro create_external_schema(source_node) -%}
+    {{ adapter.dispatch('create_external_schema', 'dbt')(source_node) }}
+{%- endmacro -%}
+
+{%- macro default__create_external_schema(source_node) -%}
+    {%- set fqn -%}
+        {%- if source_node.database -%}
+            {{ source_node.database }}.{{ source_node.schema }}
+        {%- else -%}
+            {{ source_node.schema }}
+        {%- endif -%}
+    {%- endset -%}
+
+    {%- set ddl -%}
+        create schema if not exists {{ fqn }}
+    {%- endset -%}
+
+    {{ return(ddl) }}
+{%- endmacro -%}
+
+
+{% macro exit_transaction() %}
+    {{ return(adapter.dispatch('exit_transaction', 'dbt')()) }}
+{% endmacro %}
+
+{% macro default__exit_transaction() %}
+    {{ return('') }}
+{% endmacro %}
+
+{% macro dropif(node) %}
+    {{ adapter.dispatch('dropif', 'dbt')(node) }}
+{% endmacro %}
+
+{% macro default__dropif() %}
+    {{ exceptions.raise_compiler_error(
+        "Dropping external tables is not implemented for the default adapter"
+    ) }}
+{% endmacro %}