Merge pull request #316 from godatadriven/support-artifacts-0.8

Support dbt-artifacts-parser>=0.8
godatadriven · Dec 19, 2024 · 68b9f10 · 68b9f10
2 parents 03f81ad + 9e117a8
commit 68b9f10
Show file tree

Hide file tree

Showing 10 changed files with 284 additions and 278 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -83,7 +83,7 @@ version = "0.0.0"
 [tool.poetry.dependencies]
 python = ">=3.9,<3.14"
 click = "<9"
-dbt-artifacts-parser = "<0.8"
+dbt-artifacts-parser = ">=0.8"
 jinja2 = "^3"
 jinja2-simple-tags = "^0"
 levenshtein = ">=0.26.1"
@@ -165,7 +165,7 @@ indent-width = 4
 target-version = "py38"
 
 [tool.ruff.lint]
-ignore = ["COM812", "D100", "D203", "D213", "D406", "D407", "D409", "E501", "G004", "S101"]
+ignore = ["COM812", "D100", "D203", "D213", "D406", "D407", "D409", "E501", "G004", "LOG015", "S101"]
 preview=true
 select = ["A", "ARG", "B", "C4", "COM", "D", "DOC", "E", "EXE", "F", "I", "ICN", "LOG", "G", "N", "PT", "PTH", "Q", "RUF", "S", "SIM", "T20", "TCH", "TID", "W"]
 unfixable = ["B"]

diff --git a/scripts/generate_artifacts.py b/scripts/generate_artifacts.py
@@ -9,7 +9,7 @@
 def build_pex_file(dbt_version, pex_file_name):
     """Build a pex file of `dbt-bouncer` if it does not already exist."""
     if not Path(pex_file_name).exists():
-        logging.info(f"Building pex file for dbt version {dbt_version}")  # noqa: LOG015
+        logging.info(f"Building pex file for dbt version {dbt_version}")
         sh.poetry(
             [
                 "run",
@@ -42,7 +42,7 @@ def generate_artifacts(
     pex_file_name,
 ):
     """Generate dbt artifacts for all specified versions of dbt. These artifacts are used in testing."""
-    logging.info(f"Generating dbt artifacts for dbt version {dbt_version}")  # noqa: LOG015
+    logging.info(f"Generating dbt artifacts for dbt version {dbt_version}")
     Path(artifact_path).mkdir(exist_ok=True, parents=True)
     sh.python(
         [

diff --git a/src/dbt_bouncer/artifact_parsers/parsers_catalog.py b/src/dbt_bouncer/artifact_parsers/parsers_catalog.py
@@ -9,9 +9,11 @@
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=UserWarning)
     from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-        CatalogTable,
         CatalogV1,
     )
+    from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
+        Nodes as CatalogNodes,
+    )
 if TYPE_CHECKING:
     from pathlib import Path
 
@@ -24,7 +26,7 @@
 class DbtBouncerCatalogNode(BaseModel):
     """Model for all nodes in `catalog.json`."""
 
-    catalog_node: CatalogTable
+    catalog_node: CatalogNodes
     original_file_path: str
     unique_id: str
 

diff --git a/src/dbt_bouncer/checks/catalog/check_catalog_sources.py b/src/dbt_bouncer/checks/catalog/check_catalog_sources.py
@@ -13,14 +13,16 @@
 
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=UserWarning)
-        from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogTable
+        from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
+            Nodes as CatalogNodes,
+        )
 
 
 class CheckSourceColumnsAreAllDocumented(BaseCheck):
     """All columns in a source should be included in the source's properties file, i.e. `.yml` file.
 
     Receives:
-        catalog_source (CatalogTable): The CatalogTable object to check.
+        catalog_source (CatalogNodes): The CatalogNodes object to check.
         sources (List[DbtBouncerSourceBase]): List of DbtBouncerSourceBase objects parsed from `catalog.json`.
 
     Other Parameters:
@@ -36,7 +38,7 @@ class CheckSourceColumnsAreAllDocumented(BaseCheck):
 
     """
 
-    catalog_source: "CatalogTable" = Field(default=None)
+    catalog_source: "CatalogNodes" = Field(default=None)
     name: Literal["check_source_columns_are_all_documented"]
     sources: List["DbtBouncerSourceBase"] = Field(default=[])
 

diff --git a/src/dbt_bouncer/checks/catalog/check_columns.py b/src/dbt_bouncer/checks/catalog/check_columns.py
@@ -8,7 +8,9 @@
 
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=UserWarning)
-        from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogTable
+        from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
+            Nodes as CatalogNodes,
+        )
     from dbt_bouncer.artifact_parsers.parsers_manifest import (
         DbtBouncerModelBase,
         DbtBouncerTestBase,
@@ -23,7 +25,7 @@ class CheckColumnDescriptionPopulated(BaseCheck):
     """Columns must have a populated description.
 
     Receives:
-        catalog_node (CatalogTable): The CatalogTable object to check.
+        catalog_node (CatalogNodes): The CatalogNodes object to check.
         models (List[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
 
     Other Parameters:
@@ -40,7 +42,7 @@ class CheckColumnDescriptionPopulated(BaseCheck):
 
     """
 
-    catalog_node: "CatalogTable" = Field(default=None)
+    catalog_node: "CatalogNodes" = Field(default=None)
     models: List["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_column_description_populated"]
 
@@ -69,7 +71,7 @@ class CheckColumnHasSpecifiedTest(BaseCheck):
         test_name (str): Name of the test to check for.
 
     Receives:
-        catalog_node (CatalogTable): The CatalogTable object to check.
+        catalog_node (CatalogNodes): The CatalogNodes object to check.
         tests (List[DbtBouncerTestBase]): List of DbtBouncerTestBase objects parsed from `manifest.json`.
 
     Other Parameters:
@@ -87,7 +89,7 @@ class CheckColumnHasSpecifiedTest(BaseCheck):
 
     """
 
-    catalog_node: "CatalogTable" = Field(default=None)
+    catalog_node: "CatalogNodes" = Field(default=None)
     column_name_pattern: str
     name: Literal["check_column_has_specified_test"]
     test_name: str
@@ -126,7 +128,7 @@ class CheckColumnNameCompliesToColumnType(BaseCheck):
         types (List[str]): List of data types to check.
 
     Receives:
-        catalog_node (CatalogTable): The CatalogTable object to check.
+        catalog_node (CatalogNodes): The CatalogNodes object to check.
 
     Other Parameters:
         exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
@@ -166,7 +168,7 @@ class CheckColumnNameCompliesToColumnType(BaseCheck):
 
     """
 
-    catalog_node: "CatalogTable" = Field(default=None)
+    catalog_node: "CatalogNodes" = Field(default=None)
     column_name_pattern: str
     name: Literal["check_column_name_complies_to_column_type"]
     types: List[str]
@@ -187,7 +189,7 @@ class CheckColumnsAreAllDocumented(BaseCheck):
     """All columns in a model should be included in the model's properties file, i.e. `.yml` file.
 
     Receives:
-        catalog_node (CatalogTable): The CatalogTable object to check.
+        catalog_node (CatalogNodes): The CatalogNodes object to check.
         models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.
 
     Other Parameters:
@@ -203,7 +205,7 @@ class CheckColumnsAreAllDocumented(BaseCheck):
 
     """
 
-    catalog_node: "CatalogTable" = Field(default=None)
+    catalog_node: "CatalogNodes" = Field(default=None)
     models: List["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_columns_are_all_documented"]
 
@@ -225,7 +227,7 @@ class CheckColumnsAreDocumentedInPublicModels(BaseCheck):
     """Columns should have a populated description in public models.
 
     Receives:
-        catalog_node (CatalogTable): The CatalogTable object to check.
+        catalog_node (CatalogNodes): The CatalogNodes object to check.
         models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.
 
     Other Parameters:
@@ -241,7 +243,7 @@ class CheckColumnsAreDocumentedInPublicModels(BaseCheck):
 
     """
 
-    catalog_node: "CatalogTable" = Field(default=None)
+    catalog_node: "CatalogNodes" = Field(default=None)
     models: List["DbtBouncerModelBase"] = Field(default=[])
     name: Literal["check_columns_are_documented_in_public_models"]
 

diff --git a/src/dbt_bouncer/config_file_validator.py b/src/dbt_bouncer/config_file_validator.py
@@ -226,7 +226,7 @@ def validate_conf(
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", category=UserWarning)
             from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-                CatalogTable,  # noqa: F401
+                Nodes as CatalogNodes,  # noqa: F401
             )
     if "manifest_checks" in check_categories:
         import dbt_bouncer.checks.manifest

diff --git a/tests/unit/checks/catalog/test_catalog_sources.py b/tests/unit/checks/catalog/test_catalog_sources.py
@@ -5,7 +5,7 @@
 
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=UserWarning)
-    from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogTable
+    from dbt_artifacts_parser.parsers.catalog.catalog_v1 import Nodes as CatalogNodes
 
 from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import Sources
 from dbt_bouncer.artifact_parsers.parsers_manifest import (
@@ -22,7 +22,7 @@
     ("catalog_source", "sources", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -74,7 +74,7 @@
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {

diff --git a/tests/unit/checks/catalog/test_columns.py b/tests/unit/checks/catalog/test_columns.py
@@ -5,7 +5,7 @@
 
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=UserWarning)
-    from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogTable
+    from dbt_artifacts_parser.parsers.catalog.catalog_v1 import Nodes as CatalogNodes
 
 
 from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import Nodes4, Nodes6
@@ -34,7 +34,7 @@
     ("catalog_node", "models", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -90,7 +90,7 @@
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -159,7 +159,7 @@ def test_check_column_description_populated(catalog_node, models, expectation):
     ("catalog_node", "column_name_pattern", "test_name", "tests", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -208,7 +208,7 @@ def test_check_column_description_populated(catalog_node, models, expectation):
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -279,7 +279,7 @@ def test_check_column_has_specified_test(
     ("catalog_node", "models", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -333,7 +333,7 @@ def test_check_column_has_specified_test(
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -396,7 +396,7 @@ def test_check_columns_are_all_documented(catalog_node, models, expectation):
     ("catalog_node", "models", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -442,7 +442,7 @@ def test_check_columns_are_all_documented(catalog_node, models, expectation):
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {
@@ -517,7 +517,7 @@ def test_check_columns_are_documented_in_public_models(
     ("catalog_node", "column_name_pattern", "types", "expectation"),
     [
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1_date": {
@@ -540,7 +540,7 @@ def test_check_columns_are_documented_in_public_models(
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1_date": {
@@ -573,7 +573,7 @@ def test_check_columns_are_documented_in_public_models(
             does_not_raise(),
         ),
         (
-            CatalogTable(
+            CatalogNodes(
                 **{
                     "columns": {
                         "col_1": {

diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py
@@ -5,7 +5,7 @@
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=UserWarning)
     from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-        CatalogTable,  # noqa: F401
+        Nodes as CatalogNodes,  # noqa: F401
     )
 from unittest.mock import MagicMock
 
@@ -55,7 +55,7 @@ def test_runner_coverage(caplog, tmp_path):
             warnings.filterwarnings("ignore", category=UserWarning)
             from dbt_artifacts_parser.parser import parse_manifest
             from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-                CatalogTable,  # noqa: F401
+                Nodes as CatalogNodes,  # noqa: F401
             )
         from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (  # noqa: F401
             Exposures,
@@ -189,7 +189,7 @@ def test_runner_failure():
             warnings.filterwarnings("ignore", category=UserWarning)
             from dbt_artifacts_parser.parser import parse_manifest
             from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-                CatalogTable,  # noqa: F401
+                Nodes as CatalogNodes,  # noqa: F401
             )
         from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (  # noqa: F401
             Exposures,
@@ -315,7 +315,7 @@ def test_runner_success():
             warnings.filterwarnings("ignore", category=UserWarning)
             from dbt_artifacts_parser.parser import parse_manifest
             from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-                CatalogTable,  # noqa: F401
+                Nodes as CatalogNodes,  # noqa: F401
             )
         from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (  # noqa: F401
             Exposures,
@@ -440,7 +440,7 @@ def test_runner_windows(caplog, tmp_path):
         warnings.filterwarnings("ignore", category=UserWarning)
         from dbt_artifacts_parser.parser import parse_manifest
         from dbt_artifacts_parser.parsers.catalog.catalog_v1 import (
-            CatalogTable,  # noqa: F401
+            Nodes as CatalogNodes,  # noqa: F401
         )
 
     DbtBouncerConf.model_rebuild()