Merge pull request #225 from godatadriven/add-windows-support

Adding support for Windows

.github/workflows/ci_pipeline.yml

@@ -11,6 +11,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  DBT_PROFILES_DIR: dbt_project
+  DBT_PROJECT_DIR: dbt_project
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   IMAGE_NAME: ${{ github.repository }}
   POETRY_VERSION: "1.8.3"
@@ -50,7 +52,7 @@ jobs:
 
     unit-tests:
         needs: [pre-commit]
-        runs-on: ubuntu-latest
+        runs-on: ${{ matrix.operating-system }}
         permissions:
             contents: write
             id-token: write
@@ -59,6 +61,9 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
+                operating-system:
+                  - ubuntu-latest
+                  - windows-latest
                 python-version:
                     - '3.8'
                     - '3.9'
@@ -69,6 +74,7 @@ jobs:
             - uses: actions/checkout@v4
 
             - name: checkout-merge
+              if: matrix.operating-system != 'windows-latest'
               uses: check-spelling/[email protected]
 
             - name: Setup Python
@@ -81,7 +87,7 @@ jobs:
               run: make test-unit | tee pytest-coverage.txt && exit ${PIPESTATUS[0]}
 
             - name: Pytest coverage comment
-              if: matrix.python-version == '3.11'
+              if: matrix.python-version == '3.11' && matrix.operating-system == 'ubuntu-latest'
               uses: MishaKav/pytest-coverage-comment@main
               with:
                   pytest-coverage-path: ./pytest-coverage.txt
@@ -92,6 +98,15 @@ jobs:
             - name: Run pytest (integration tests)
               run: make test-integration
 
+            - name: Regenerate dbt artifacts
+              run: |
+                poetry run dbt deps
+                poetry run dbt build
+                poetry run dbt docs generate
+
+            - name: Run `dbt-bouncer` on generated artifacts
+              run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml
+
     e2e-tests:
         needs: [pre-commit]
         runs-on: ubuntu-latest

.github/workflows/merge_pipeline.yml

@@ -7,6 +7,8 @@
         - main
 
   env:
+    DBT_PROFILES_DIR: dbt_project
+    DBT_PROJECT_DIR: dbt_project
     POETRY_VERSION: "1.8.3"
     POETRY_VIRTUALENVS_IN_PROJECT: true
 
@@ -122,10 +124,13 @@
               run: python ./dist/dbt-bouncer.pex --config-file dbt-bouncer-example.yml
 
       pypi-tests:
-        runs-on: ubuntu-latest
+        runs-on: ${{ matrix.operating-system }}
         strategy:
             fail-fast: false
             matrix:
+                operating-system:
+                  - ubuntu-latest
+                  - windows-latest
                 python-version:
                     - '3.8'
                     - '3.9'
@@ -154,3 +159,12 @@
 
             - name: Run `dbt-bouncer`
               run: dbt-bouncer --config-file dbt-bouncer-example.yml
+
+            - name: Regenerate dbt artifacts
+              run: |
+                poetry run dbt deps
+                poetry run dbt build
+                poetry run dbt docs generate
+
+            - name: Run `dbt-bouncer` on generated artifacts
+              run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml

.github/workflows/post_release_pipeline.yml

@@ -6,6 +6,8 @@ on:
         types: [published]
 
 env:
+    DBT_PROFILES_DIR: dbt_project
+    DBT_PROJECT_DIR: dbt_project
     POETRY_VERSION: "1.8.3"
     POETRY_VIRTUALENVS_IN_PROJECT: true
 
@@ -77,10 +79,13 @@ jobs:
 
     pypi-tests:
         needs: [pypi-pause]
-        runs-on: ubuntu-latest
+        runs-on: ${{ matrix.operating-system }}
         strategy:
             fail-fast: false
             matrix:
+                operating-system:
+                  - ubuntu-latest
+                  - windows-latest
                 python-version:
                     - '3.8'
                     - '3.9'
@@ -109,3 +114,12 @@ jobs:
 
             - name: Run `dbt-bouncer`
               run: dbt-bouncer --config-file dbt-bouncer-example.yml
+
+            - name: Regenerate dbt artifacts
+              run: |
+                poetry run dbt deps
+                poetry run dbt build
+                poetry run dbt docs generate
+
+            - name: Run `dbt-bouncer` on generated artifacts
+              run: poetry run dbt-bouncer --config-file ./dbt-bouncer-example.yml

docs/config_file.md

@@ -88,6 +88,22 @@ To determine if a check accepts these arguments view the [Checks page](./checks/
         model_name_pattern: ^stg_
     ```
 
+!!! note
+
+    When compiled on Windows machines, keys such as `original_file_path`, `patch_path` and `path` take the form:
+
+    ```shell
+    models\\staging\\crm\\model_1.sql
+    ```
+
+    When compiled on Linux and Mac machines, these same keys take the form:
+
+    ```shell
+    models/staging/crm/model_1.sql
+    ```
+
+    `dbt-bouncer` converts all of these paths to the Linux/Mac form, hence when you are supplying values to `exclude` and `include` you should use the Linux/Mac form.
+
 ### Severity
 
 All checks accept a `severity` argument, valid values are:

src/dbt_bouncer/checks/manifest/check_lineage.py

@@ -13,6 +13,8 @@
 
 from pydantic import Field
 
+from dbt_bouncer.utils import clean_path_str
+
 
 class CheckLineagePermittedUpstreamModels(BaseCheck):
     """Upstream models must have a path that matches the provided `upstream_path_pattern`.
@@ -64,9 +66,11 @@ def execute(self) -> None:
             upstream_model
             for upstream_model in upstream_models
             if re.compile(self.upstream_path_pattern.strip()).match(
-                next(
-                    m for m in self.models if m.unique_id == upstream_model
-                ).original_file_path,
+                clean_path_str(
+                    next(
+                        m for m in self.models if m.unique_id == upstream_model
+                    ).original_file_path
+                ),
             )
             is None
         ]

src/dbt_bouncer/checks/manifest/check_macros.py

@@ -5,6 +5,8 @@
 
 from pydantic import Field
 
+from dbt_bouncer.utils import clean_path_str
+
 if TYPE_CHECKING:
     import warnings
 
@@ -239,12 +241,16 @@ def execute(self) -> None:
         if self.macro.name.startswith("test_"):
             assert (
                 self.macro.name[5:]
-                == self.macro.original_file_path.split("/")[-1].split(".")[0]
+                == clean_path_str(self.macro.original_file_path)
+                .split("/")[-1]
+                .split(".")[0]
             ), f"Macro `{self.macro.unique_id}` is not in a file named `{self.macro.name[5:]}.sql`."
         else:
             assert (
                 self.macro.name
-                == self.macro.original_file_path.split("/")[-1].split(".")[0]
+                == clean_path_str(self.macro.original_file_path)
+                .split("/")[-1]
+                .split(".")[0]
             ), f"Macro `{self.macro.name}` is not in a file of the same name."
 
 
@@ -272,14 +278,16 @@ class CheckMacroPropertyFileLocation(BaseCheck):
 
     def execute(self) -> None:
         """Execute the check."""
-        expected_substr = "_".join(self.macro.original_file_path[6:].split("/")[:-1])
+        expected_substr = "_".join(
+            clean_path_str(self.macro.original_file_path)[6:].split("/")[:-1]
+        )
 
         assert (
-            self.macro.patch_path is not None
+            clean_path_str(self.macro.patch_path) is not None
         ), f"Macro `{self.macro.name}` is not defined in a `.yml` properties file."
-        properties_yml_name = self.macro.patch_path.split("/")[-1]
+        properties_yml_name = clean_path_str(self.macro.patch_path).split("/")[-1]
 
-        if self.macro.original_file_path.startswith(
+        if clean_path_str(self.macro.original_file_path).startswith(
             "tests/",
         ):  # Do not check generic tests (which are also macros)
             pass

src/dbt_bouncer/checks/manifest/check_models.py

@@ -30,6 +30,7 @@
         DbtBouncerManifest,
         DbtBouncerModelBase,
     )
+from dbt_bouncer.utils import clean_path_str
 
 
 class CheckModelAccess(BaseCheck):
@@ -214,14 +215,17 @@ class CheckModelDocumentedInSameDirectory(BaseCheck):
 
     def execute(self) -> None:
         """Execute the check."""
-        model_sql_dir = self.model.original_file_path.split("/")[:-1]
+        model_sql_dir = clean_path_str(self.model.original_file_path).split("/")[:-1]
         assert (  # noqa: PT018
-            hasattr(self.model, "patch_path") and self.model.patch_path is not None
+            hasattr(self.model, "patch_path")
+            and clean_path_str(self.model.patch_path) is not None
         ), f"`{self.model.name}` is not documented."
 
-        model_doc_dir = self.model.patch_path[
-            self.model.patch_path.find("models") :
-        ].split("/")[:-1]
+        model_doc_dir = clean_path_str(
+            self.model.patch_path[
+                clean_path_str(self.model.patch_path).find("models") :
+            ]
+        ).split("/")[:-1]
 
         assert (
             model_doc_dir == model_sql_dir
@@ -337,9 +341,11 @@ class CheckModelDirectories(BaseCheck):
     def execute(self) -> None:
         """Execute the check."""
         matched_path = re.compile(self.include.strip()).match(
-            self.model.original_file_path
+            clean_path_str(self.model.original_file_path)
         )
-        path_after_match = self.model.original_file_path[matched_path.end() + 1 :]  # type: ignore[union-attr]
+        path_after_match = clean_path_str(self.model.original_file_path)[
+            matched_path.end() + 1 :
+        ]
 
         assert (
             path_after_match.split("/")[0] in self.permitted_sub_directories
@@ -908,16 +914,17 @@ class CheckModelPropertyFileLocation(BaseCheck):
     def execute(self) -> None:
         """Execute the check."""
         assert (  # noqa: PT018
-            hasattr(self.model, "patch_path") and self.model.patch_path is not None
+            hasattr(self.model, "patch_path")
+            and clean_path_str(self.model.patch_path) is not None
         ), f"`{self.model.name}` is not documented."
 
         expected_substr = (
-            "_".join(self.model.original_file_path.split("/")[1:-1])
+            "_".join(clean_path_str(self.model.original_file_path).split("/")[1:-1])
             .replace("staging", "stg")
             .replace("intermediate", "int")
             .replace("marts", "")
         )
-        properties_yml_name = self.model.patch_path.split("/")[-1]
+        properties_yml_name = clean_path_str(self.model.patch_path).split("/")[-1]
 
         assert properties_yml_name.startswith(
             "_",

src/dbt_bouncer/checks/manifest/check_sources.py

@@ -14,7 +14,7 @@
 from pydantic import Field
 
 from dbt_bouncer.check_base import BaseCheck
-from dbt_bouncer.utils import find_missing_meta_keys
+from dbt_bouncer.utils import clean_path_str, find_missing_meta_keys
 
 
 class CheckSourceDescriptionPopulated(BaseCheck):
@@ -280,7 +280,9 @@ class CheckSourcePropertyFileLocation(BaseCheck):
 
     def execute(self) -> None:
         """Execute the check."""
-        path_cleaned = self.source.original_file_path.replace("models/staging", "")
+        path_cleaned = clean_path_str(self.source.original_file_path).replace(
+            "models/staging", ""
+        )
         expected_substring = "_".join(path_cleaned.split("/")[:-1])
 
         assert path_cleaned.split(

src/dbt_bouncer/config_file_parser.py

@@ -1,8 +1,11 @@
+from pathlib import Path
 from typing import Any, ClassVar, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import Annotated
 
+from dbt_bouncer.utils import clean_path_str
+
 
 class DbtBouncerConf(BaseModel):
     """Base model for the config file contents."""
@@ -11,16 +14,17 @@ class DbtBouncerConf(BaseModel):
 
     from dbt_bouncer.utils import get_check_objects
 
-    check_classes: List[Dict[str, Union[Any, str]]] = [
-        {"class": getattr(x, x.__name__), "source_file": x.__file__}
+    check_classes: List[Dict[str, Union[Any, Path]]] = [
+        {
+            "class": getattr(x, x.__name__),
+            "source_file": Path(clean_path_str(x.__file__)),
+        }
         for x in get_check_objects()
     ]
 
     # Catalog checks
     catalog_check_classes: ClassVar = [
-        x["class"]
-        for x in check_classes
-        if x["source_file"].split("/")[-2] == "catalog"
+        x["class"] for x in check_classes if x["source_file"].parts[-2] == "catalog"
     ]
 
     CatalogCheckConfigs: ClassVar = Annotated[
@@ -30,9 +34,7 @@ class DbtBouncerConf(BaseModel):
 
     # Manifest checks
     manifest_check_classes: ClassVar = [
-        x["class"]
-        for x in check_classes
-        if x["source_file"].split("/")[-2] == "manifest"
+        x["class"] for x in check_classes if x["source_file"].parts[-2] == "manifest"
     ]
 
     ManifestCheckConfigs: ClassVar = Annotated[
@@ -42,9 +44,7 @@ class DbtBouncerConf(BaseModel):
 
     # Run result checks
     run_results_check_classes: ClassVar = [
-        x["class"]
-        for x in check_classes
-        if x["source_file"].split("/")[-2] == "run_results"
+        x["class"] for x in check_classes if x["source_file"].parts[-2] == "run_results"
     ]
 
     RunResultsCheckConfigs: ClassVar = Annotated[