v1.6.0rc1

.github/workflows/integration-tests-azure.yml

@@ -14,9 +14,16 @@ jobs:
     name: Integration tests on Azure
     strategy:
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
         msodbc_version: ["17", "18"]
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      deployments: read
+      packages: none
+      pull-requests: write
+      security-events: write
     container:
       image: ghcr.io/dbt-msft/dbt-sqlserver:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }}
     steps:

CHANGELOG.md

@@ -1,4 +1,43 @@
 # Changelog
+## v1.6.0rc1
+
+* Support for [dbt-core 1.6](https://github.com/dbt-labs/dbt-core/releases/tag/v1.6.0)
+
+#### Breaking Changes
+* Dropped support for Python 3.7 ([#7082](https://github.com/dbt-labs/dbt-core/issues/7082https://github.com/dbt-labs/dbt-core/issues/7082))
+
+## Features
+* Add support for materialized views ([#6911](https://github.com/dbt-labs/dbt-core/issues/6911))
+  * important note! unlike [dbt's materialized view](https://docs.getdbt.com/docs/build/materializations), [Synapse's materialized view](https://learn.microsoft.com/en-us/sql/t-sql/statements/create-materialized-view-as-select-transact-sql?view=azure-sqldw-latest&context=%2Fazure%2Fsynapse-analytics%2Fcontext%2Fcontext) must be created using aggregation and/or "GROUP BY"! 
+* ~~dbt clone ([#7258](https://github.com/dbt-labs/dbt-core/issues/7258)~~ Synapse does not support CLONE)
+* Revamp dbt debug ([#7104](https://github.com/dbt-labs/dbt-core/issues/7104))
+* Added new adapter zone tests
+  - constraints
+  - null_compare
+  - validate_sql
+  - equals
+  - dbt_clone
+
+## v.1.5.0rc1
+
+* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0)
+  * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros
+  * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class.
+  * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse
+  * Add adapter tests zones
+    - caching
+    - column_types
+    - constraints
+    - hooks
+    - simple_copy
+
+## v1.4.1rc1
+
+#### Under the hood
+* Switch dependency from dbt-sqlserver to dbt-fabric [#441](https://github.com/dbt-msft/dbt-sqlserver/issues/441)
+  * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt
+  * [Stackoverflow](https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle) about pyodbc "Symbol not found: _SQLAllocHandle" error
+
 ## v1.4.0
 
 #### Features

CONTRIBUTING.md

@@ -16,7 +16,7 @@ pyenv activate dbt-synapse
 Install the development dependencies and pre-commit and get information about possible make commands:
 
 ```shell
-make dev
+make dev  # for Mac users, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt before running `make dev`
 make help
 ```
 

dbt/adapters/synapse/__init__.py

@@ -1,6 +1,7 @@
 from dbt.adapters.base import AdapterPlugin
 
 from dbt.adapters.synapse.synapse_adapter import SynapseAdapter
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 from dbt.adapters.synapse.synapse_credentials import SynapseCredentials
 from dbt.include import synapse
@@ -9,7 +10,13 @@
     adapter=SynapseAdapter,
     credentials=SynapseCredentials,
     include_path=synapse.PACKAGE_PATH,
-    dependencies=["sqlserver"],
+    dependencies=["fabric"],
 )
 
-__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"]
+__all__ = [
+    "Plugin",
+    "SynapseConnectionManager",
+    "SynapseColumn",
+    "SynapseAdapter",
+    "SynapseCredentials",
+]

dbt/adapters/synapse/__version__.py

@@ -1 +1 @@
-version = "1.4.0"
+version = "1.6.0rc1"

dbt/adapters/synapse/synapse_adapter.py

@@ -1,7 +1,81 @@
-from dbt.adapters.sqlserver import SQLServerAdapter
+from enum import Enum
+from typing import Any, Dict, List, Optional
 
+from dbt.adapters.base.relation import BaseRelation
+from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.fabric import FabricAdapter
+from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType
+from dbt.events.functions import fire_event
+from dbt.events.types import SchemaCreation
+
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 
 
-class SynapseAdapter(SQLServerAdapter):
+class SynapseAdapter(FabricAdapter):
     ConnectionManager = SynapseConnectionManager
+    Column = SynapseColumn
+
+    def create_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
+        macro_name = CREATE_SCHEMA_MACRO_NAME
+        kwargs = {
+            "relation": relation,
+        }
+
+        if self.config.credentials.schema_authorization:
+            kwargs["schema_authorization"] = self.config.credentials.schema_authorization
+            macro_name = "synapse__create_schema_with_authorization"
+
+        self.execute_macro(macro_name, kwargs=kwargs)
+        self.commit_if_has_connection()
+
+    class ConstraintSupport(str, Enum):
+        ENFORCED = "enforced"
+        NOT_ENFORCED = "not_enforced"
+        NOT_SUPPORTED = "not_supported"
+
+    # https://learn.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-table-constraints#table-constraints
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,  # no CHECK support for Synapse
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,  # no FK support for Synapse
+    }
+
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text.
+        Should be overriden by adapters which need custom constraint rendering."""
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"check {constraint.expression}"
+        elif constraint.type == ConstraintType.not_null:
+            return "not null"
+        elif constraint.type == ConstraintType.unique:
+            return "unique NOT ENFORCED"
+        elif constraint.type == ConstraintType.primary_key:
+            return "primary key NONCLUSTERED NOT ENFORCED"
+        elif constraint.type == ConstraintType.foreign_key:
+            return "foreign key"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return constraint.expression
+        else:
+            return None
+
+    @classmethod
+    def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
+        rendered_column_constraints = []
+
+        for v in raw_columns.values():
+            rendered_column_constraint = [f"[{v['name']}] {v['data_type']}"]
+            for con in v.get("constraints", None):
+                constraint = cls._parse_column_constraint(con)
+                c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
+                if c is not None:
+                    rendered_column_constraint.append(c)
+            rendered_column_constraints.append(" ".join(rendered_column_constraint))
+
+        return rendered_column_constraints

dbt/adapters/synapse/synapse_column.py

@@ -0,0 +1,16 @@
+from dbt.adapters.fabric import FabricColumn
+
+
+class SynapseColumn(FabricColumn):
+    # extending list of integer types for synapse
+    def is_integer(self) -> bool:
+        return self.dtype.lower() in [
+            # real types
+            "smallint",
+            "bigint",
+            "tinyint",
+            "serial",
+            "bigserial",
+            "int",
+            "bit",
+        ]

dbt/adapters/synapse/synapse_connection_manager.py

@@ -1,6 +1,6 @@
-from dbt.adapters.sqlserver import SQLServerConnectionManager
+from dbt.adapters.fabric import FabricConnectionManager
 
 
-class SynapseConnectionManager(SQLServerConnectionManager):
+class SynapseConnectionManager(FabricConnectionManager):
     TYPE = "synapse"
     TOKEN = None

dbt/adapters/synapse/synapse_credentials.py

@@ -1,10 +1,10 @@
 from dataclasses import dataclass
 
-from dbt.adapters.sqlserver import SQLServerCredentials
+from dbt.adapters.fabric import FabricCredentials
 
 
 @dataclass
-class SynapseCredentials(SQLServerCredentials):
+class SynapseCredentials(FabricCredentials):
     @property
     def type(self):
         return "synapse"

dbt/include/synapse/macros/adapters/indexes.sql

@@ -126,3 +126,35 @@ declare @drop_remaining_indexes_last nvarchar(max) = (
 {% macro create_nonclustered_index(columns, includes=False) %}
   {{ return(create_nonclustered_index(columns, includes=False)) }}
 {% endmacro %}
+
+
+{% macro drop_fk_indexes_on_table(relation) -%}
+  {% call statement('find_references', fetch_result=true) %}
+      SELECT  obj.name AS FK_NAME,
+      sch.name AS [schema_name],
+      tab1.name AS [table],
+      col1.name AS [column],
+      tab2.name AS [referenced_table],
+      col2.name AS [referenced_column]
+      FROM sys.foreign_key_columns fkc
+      INNER JOIN sys.objects obj
+          ON obj.object_id = fkc.constraint_object_id
+      INNER JOIN sys.tables tab1
+          ON tab1.object_id = fkc.parent_object_id
+      INNER JOIN sys.schemas sch
+          ON tab1.schema_id = sch.schema_id
+      INNER JOIN sys.columns col1
+          ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id
+      INNER JOIN sys.tables tab2
+          ON tab2.object_id = fkc.referenced_object_id
+      INNER JOIN sys.columns col2
+          ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id
+      WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}'
+  {% endcall %}
+      {% set references = load_result('find_references')['data'] %}
+      {% for reference in references -%}
+        {% call statement('main') -%}
+           alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}]
+        {%- endcall %}
+      {% endfor %}
+{% endmacro %}

dbt/include/synapse/macros/adapters/persist_docs.sql

@@ -0,0 +1,4 @@
+{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer
+  https://github.com/dbt-msft/dbt-sqlserver/issues/134
+  https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16
+ #}

dbt/include/synapse/macros/adapters/relation.sql

@@ -5,22 +5,51 @@
 {% endmacro %}
 
 {% macro synapse__drop_relation_script(relation) -%}
-  {% if relation.type == 'view' -%}
+  {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
     {% set object_id_type = 'V' %}
   {% elif relation.type == 'table'%}
     {% set object_id_type = 'U' %}
   {%- else -%} invalid target name
   {% endif %}
+
   if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
+  {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
+    begin
+    drop view {{ relation.include(database=False) }}
+    end
+  {% elif relation.type == 'table' %}
     begin
     drop {{ relation.type }} {{ relation.include(database=False) }}
     end
+  {% endif %}
 {% endmacro %}
 
-
 {% macro synapse__rename_relation(from_relation, to_relation) -%}
-  {% call statement('rename_relation') -%}
+  {# dbt needs this 'call' macro, but it overwrites other SQL when reused in other macros #}
+  {# so '_script' macro is reuseable script, for other macros to combine with more SQL #}
 
-    rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }}
+  {% call statement('rename_relation') %}
+    {{ synapse__rename_relation_script(from_relation, to_relation) }}
   {%- endcall %}
 {% endmacro %}
+
+{% macro synapse__rename_relation_script(from_relation, to_relation) -%}
+  -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted"
+  if object_id ('{{ to_relation.include(database=False) }}','V') is not null
+    begin
+    drop view {{ to_relation.include(database=False) }}
+    end
+
+  if object_id ('{{ to_relation.include(database=False) }}','U') is not null
+    begin
+    drop table {{ to_relation.include(database=False) }}
+    end
+
+  rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }}
+{% endmacro %}
+
+{% macro synapse__truncate_relation(relation) %}
+    {% call statement('truncate_relation') -%}
+      truncate table {{ relation }}
+    {%- endcall %}
+{% endmacro %}

dbt/include/synapse/macros/adapters/schema.sql

@@ -6,3 +6,32 @@
     END
   {% endcall %}
 {% endmacro %}
+
+{% macro synapse__create_schema_with_authorization(relation, schema_authorization) -%}
+  {% call statement('create_schema') -%}
+    IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
+    BEGIN
+    EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]')
+    END
+  {% endcall %}
+{% endmacro %}
+
+{% macro synapse__drop_schema(relation) -%}
+  {%- set relations_in_schema = list_relations_without_caching(relation) %}
+
+  {% for row in relations_in_schema %}
+    {%- set schema_relation = api.Relation.create(database=relation.database,
+                                               schema=relation.schema,
+                                               identifier=row[1],
+                                               type=row[3]
+                                               ) -%}
+    {% do drop_relation(schema_relation) %}
+  {%- endfor %}
+
+  {% call statement('drop_schema') -%}
+    IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}')
+    BEGIN
+    EXEC('DROP SCHEMA [{{ relation.without_identifier().schema }}]')
+    END
+  {% endcall %}
+{% endmacro %}

dbt/include/synapse/macros/adapters/show.sql

@@ -0,0 +1,22 @@
+{% macro get_show_sql(compiled_code, sql_header, limit) -%}
+  {%- if sql_header -%}
+  {{ sql_header }}
+  {%- endif -%}
+  {%- if limit is not none -%}
+  {{ get_limit_subquery_sql(compiled_code, limit) }}
+  {%- else -%}
+  {{ compiled_code }}
+  {%- endif -%}
+{% endmacro %}
+
+{% macro get_limit_subquery_sql(sql, limit) %}
+  {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}
+{% endmacro %}
+
+{# Synapse doesnt support ANSI LIMIT clause #}
+{% macro synapse__get_limit_subquery_sql(sql, limit) %}
+    select top {{ limit }} *
+    from (
+        {{ sql }}
+    ) as model_limit_subq
+{% endmacro %}