Semantic models should be based on public models only.
+
+
+
Parameters:
+
+
+
+
Name
+
Type
+
Description
+
Default
+
+
+
+
+
models
+
+ List[DbtBouncerModel]
+
+
+
+
List of DbtBouncerModel objects parsed from manifest.json.
+
+
+
+ required
+
+
+
+
semantic_model
+
+ DbtBouncerSemanticModel
+
+
+
+
The DbtBouncerSemanticModel object to check.
+
+
+
+ required
+
+
+
+
+
Other Parameters (passed via config file):
+
+
+
+
Name
+
Type
+
Description
+
+
+
+
+
exclude
+
+ Optional[str]
+
+
+
+
Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Semantic model paths that match the pattern will not be checked.
+
+
+
+
+
include
+
+ Optional[str]
+
+
+
+
Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Only semantic model paths that match the pattern will be checked.
defcheck_semantic_model_based_on_non_public_models(
+models:List["DbtBouncerModel"],
+semantic_model:"DbtBouncerSemanticModel",
+**kwargs,
+)->None:
+"""Semantic models should be based on public models only.
+
+ Parameters:
+ models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.
+ semantic_model (DbtBouncerSemanticModel): The DbtBouncerSemanticModel object to check.
+
+ Other Parameters:
+ exclude (Optional[str]): Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Semantic model paths that match the pattern will not be checked.
+ include (Optional[str]): Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Only semantic model paths that match the pattern will be checked.
+ severity (Optional[Literal["error", "warn"]]): Severity level of the check. Default: `error`.
+
+ Example(s):
+ ```yaml
+ manifest_checks:
+ - name: check_semantic_model_based_on_non_public_models
+ ```
+
+ """
+non_public_upstream_dependencies=[]
+formodelinsemantic_model.depends_on.nodes:
+if(
+model.split(".")[0]=="model"
+andmodel.split(".")[1]==semantic_model.package_name
+):
+model=next(mforminmodelsifm.unique_id==model)
+ifmodel.access.value!="public":
+non_public_upstream_dependencies.append(model.name)
+
+assertnotnon_public_upstream_dependencies,f"Semantic model `{semantic_model.name}` is based on a model(s) that is not public: {non_public_upstream_dependencies}."
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/latest/checks/manifest/check_sources/index.html b/latest/checks/manifest/check_sources/index.html
index c4b4642a..be2e62b0 100644
--- a/latest/checks/manifest/check_sources/index.html
+++ b/latest/checks/manifest/check_sources/index.html
@@ -14,7 +14,7 @@
-
+
@@ -627,6 +627,8 @@
+
+
@@ -810,6 +812,27 @@
+
Source code in src/dbt_bouncer/checks/catalog/check_catalog_sources.py
def check_source_columns_are_all_documented(\n catalog_source: \"DbtBouncerCatalogNode\",\n sources: List[\"DbtBouncerSource\"],\n **kwargs,\n) -> None:\n \"\"\"All columns in a source should be included in the source's properties file, i.e. `.yml` file.\n\n Parameters:\n catalog_source (DbtBouncerCatalogNode): The DbtBouncerCatalogNode object to check.\n sources (List[DbtBouncerSource]): List of DbtBouncerSource objects parsed from `catalog.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_source_columns_are_all_documented\n ```\n\n \"\"\"\n source = next(s for s in sources if s.unique_id == catalog_source.unique_id)\n undocumented_columns = [\n v.name\n for _, v in catalog_source.columns.items()\n if v.name not in source.columns\n ]\n assert not undocumented_columns, f\"`{catalog_source.unique_id}` has columns that are not included in the sources properties file: {undocumented_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_description_populated(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Columns must have a populated description.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_column_description_populated\n include: ^models/marts\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n non_complying_columns = []\n for _, v in catalog_node.columns.items():\n if (\n model.columns.get(v.name) is None\n or len(model.columns[v.name].description.strip()) <= 4\n ):\n non_complying_columns.append(v.name)\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that do not have a populated description: {non_complying_columns}\"\n
Columns with specified data types must comply to the specified regexp naming pattern.
Parameters:
Name Type Description Default catalog_nodeCatalogTable
The CatalogTable object to check.
required column_name_patternstr
(str): Regex pattern to match the model name.
required typesList[str]
List of data types to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the model path. Model paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the model path. Only model paths that match the pattern will be checked.
severityOptional[Literal['error', 'warn']]
Severity level of the check. Default: error.
Example(s):
catalog_checks:\n # DATE columns must end with \"_date\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: .*_date$\n types:\n - DATE\n
catalog_checks:\n # BOOLEAN columns must start with \"is_\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^is_.*\n types:\n - BOOLEAN\n
catalog_checks:\n # Columns of all types must consist of lowercase letters and underscores. Note that the specified types depend on the underlying database.\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^[a-z_]*$\n types:\n - BIGINT\n - BOOLEAN\n - DATE\n - DOUBLE\n - INTEGER\n - VARCHAR\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_name_complies_to_column_type(\n catalog_node: \"CatalogTable\",\n column_name_pattern: str,\n types: List[str],\n **kwargs,\n) -> None:\n \"\"\"Columns with specified data types must comply to the specified regexp naming pattern.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n column_name_pattern: (str): Regex pattern to match the model name.\n types (List[str]): List of data types to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n # DATE columns must end with \"_date\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: .*_date$\n types:\n - DATE\n ```\n ```yaml\n catalog_checks:\n # BOOLEAN columns must start with \"is_\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^is_.*\n types:\n - BOOLEAN\n ```\n ```yaml\n catalog_checks:\n # Columns of all types must consist of lowercase letters and underscores. Note that the specified types depend on the underlying database.\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^[a-z_]*$\n types:\n - BIGINT\n - BOOLEAN\n - DATE\n - DOUBLE\n - INTEGER\n - VARCHAR\n ```\n\n \"\"\"\n non_complying_columns = [\n v.name\n for _, v in catalog_node.columns.items()\n if v.type in types\n and re.compile(column_name_pattern.strip()).match(v.name) is None\n ]\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that don't comply with the specified regexp pattern (`{column_name_pattern}`): {non_complying_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_columns_are_all_documented(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"All columns in a model should be included in the model's properties file, i.e. `.yml` file.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_columns_are_all_documented\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n undocumented_columns = [\n v.name\n for _, v in catalog_node.columns.items()\n if v.name not in model.columns\n ]\n assert not undocumented_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that are not included in the models properties file: {undocumented_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_columns_are_documented_in_public_models(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Columns should have a populated description in public models.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_columns_are_documented_in_public_models\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n non_complying_columns = []\n for _, v in catalog_node.columns.items():\n if model.access.value == \"public\":\n column_config = model.columns.get(v.name)\n if column_config is None or len(column_config.description.strip()) < 4:\n non_complying_columns.append(v.name)\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` is a public model but has columns that don't have a populated description: {non_complying_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_has_specified_test(\n catalog_node: \"CatalogTable\",\n column_name_pattern: str,\n test_name: str,\n tests: List[\"DbtBouncerTest\"],\n **kwargs,\n) -> None:\n \"\"\"Columns that match the specified regexp pattern must have a specified test.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n column_name_pattern (str): Regex pattern to match the column name.\n test_name (str): Name of the test to check for.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_column_has_specified_test\n column_name_pattern: ^is_.*\n test_name: not_null\n ```\n\n \"\"\"\n columns_to_check = [\n v.name\n for _, v in catalog_node.columns.items()\n if re.compile(column_name_pattern.strip()).match(v.name) is not None\n ]\n relevant_tests = [\n t\n for t in tests\n if t.test_metadata.name == test_name\n and t.attached_node == catalog_node.unique_id\n ]\n non_complying_columns = [\n c\n for c in columns_to_check\n if f\"{catalog_node.unique_id}.{c}\"\n not in [f\"{t.attached_node}.{t.column_name}\" for t in relevant_tests]\n ]\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that should have a `{test_name}` test: {non_complying_columns}\"\n
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_exposures.py
def check_exposure_based_on_non_public_models(\n exposure: \"Exposures\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Exposures should be based on public models only.\n\n Parameters:\n exposure (Exposures): The Exposures object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_non_public_models\n ```\n\n \"\"\"\n non_public_upstream_dependencies = []\n for model in exposure.depends_on.nodes:\n if (\n model.split(\".\")[0] == \"model\"\n and model.split(\".\")[1] == exposure.package_name\n ):\n model = next(m for m in models if m.unique_id == model)\n if model.access.value != \"public\":\n non_public_upstream_dependencies.append(model.name)\n\n assert not non_public_upstream_dependencies, f\"`{exposure.name}` is based on a model(s) that is not public: {non_public_upstream_dependencies}.\"\n
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_exposures.py
def check_exposure_based_on_view(\n exposure: \"Exposures\",\n models: List[\"DbtBouncerModel\"],\n materializations_to_include: List[str] = [\"ephemeral\", \"view\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Exposures should not be based on views.\n\n Parameters:\n exposure (Exposures): The Exposures object to check.\n materializations_to_include (Optional[List[str]]): List of materializations to include in the check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_view\n ```\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_view\n materializations_to_include:\n - ephemeral\n - my_custom_materialization\n - view\n ```\n\n \"\"\"\n non_table_upstream_dependencies = []\n for model in exposure.depends_on.nodes:\n if (\n model.split(\".\")[0] == \"model\"\n and model.split(\".\")[1] == exposure.package_name\n ):\n model = next(m for m in models if m.unique_id == model)\n if model.config.materialized in materializations_to_include:\n non_table_upstream_dependencies.append(model.name)\n\n assert not non_table_upstream_dependencies, f\"`{exposure.name}` is based on a model that is not a table: {non_table_upstream_dependencies}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_permitted_upstream_models(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n upstream_path_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Upstream models must have a path that matches the provided `upstream_path_pattern`.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The manifest object.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n upstream_path_pattern (str): Regexp pattern to match the upstream model(s) path.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_permitted_upstream_models\n include: ^models/staging\n upstream_path_pattern: $^\n - name: check_lineage_permitted_upstream_models\n include: ^models/intermediate\n upstream_path_pattern: ^models/staging|^models/intermediate\n - name: check_lineage_permitted_upstream_models\n include: ^models/marts\n upstream_path_pattern: ^models/staging|^models/intermediate\n ```\n\n \"\"\"\n upstream_models = [\n x\n for x in model.depends_on.nodes\n if x.split(\".\")[0] == \"model\"\n and x.split(\".\")[1] == manifest_obj.manifest.metadata.project_name\n ]\n not_permitted_upstream_models = [\n upstream_model\n for upstream_model in upstream_models\n if re.compile(upstream_path_pattern.strip()).match(\n next(m for m in models if m.unique_id == upstream_model).original_file_path,\n )\n is None\n ]\n assert not not_permitted_upstream_models, f\"`{model.name}` references upstream models that are not permitted: {[m.split('.')[-1] for m in not_permitted_upstream_models]}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_seed_cannot_be_used(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Seed cannot be referenced in models with a path that matches the specified `include` config.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_seed_cannot_be_used\n include: ^models/intermediate|^models/marts\n ```\n\n \"\"\"\n assert not [\n x for x in model.depends_on.nodes if x.split(\".\")[0] == \"seed\"\n ], f\"`{model.name}` references a seed even though this is not permitted.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_source_cannot_be_used(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Sources cannot be referenced in models with a path that matches the specified `include` config.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_source_cannot_be_used\n include: ^models/intermediate|^models/marts\n ```\n\n \"\"\"\n assert not [\n x for x in model.depends_on.nodes if x.split(\".\")[0] == \"source\"\n ], f\"`{model.name}` references a source even though this is not permitted.\"\n
# Only \"common\" macros need to have their arguments populated\nmanifest_checks:\n - name: check_macro_arguments_description_populated\n include: ^macros/common\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_arguments_description_populated(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macro arguments must have a populated description.\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_arguments_description_populated\n ```\n ```yaml\n # Only \"common\" macros need to have their arguments populated\n manifest_checks:\n - name: check_macro_arguments_description_populated\n include: ^macros/common\n ```\n\n \"\"\"\n environment = jinja2.Environment(autoescape=True, extensions=[TagExtension])\n ast = environment.parse(macro.macro_sql)\n\n # Assume macro is a \"true\" macro, if not see if it's a generic test\n try:\n macro_arguments = [a.name for a in ast.body[0].args] # type: ignore[attr-defined]\n except AttributeError:\n test_macro = next(\n x\n for x in ast.body\n if not isinstance(x.nodes[0], jinja2.nodes.Call) # type: ignore[attr-defined]\n )\n macro_arguments = [\n x.name\n for x in test_macro.nodes # type: ignore[attr-defined]\n if isinstance(x, jinja2.nodes.Name)\n ]\n\n # macro_arguments: List of args parsed from macro SQL\n # macro.arguments: List of args manually added to the properties file\n\n non_complying_args = []\n for arg in macro_arguments:\n macro_doc_raw = [x for x in macro.arguments if x.name == arg]\n if macro_doc_raw == [] or (\n arg not in [x.name for x in macro.arguments]\n or len(macro_doc_raw[0].description.strip()) <= 4\n ):\n non_complying_args.append(arg)\n\n assert (\n non_complying_args == []\n ), f\"Macro `{macro.name}` does not have a populated description for the following argument(s): {non_complying_args}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_code_does_not_contain_regexp_pattern(\n macro: \"Macros\",\n regexp_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"The raw code for a macro must not match the specified regexp pattern.\n\n Parameters:\n macro (Macros): The Macros object to check.\n regexp_pattern (str): The regexp pattern that should not be matched by the macro code.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Prefer `coalesce` over `ifnull`: https://docs.sqlfluff.com/en/stable/rules.html#sqlfluff.rules.sphinx.Rule_CV02\n - name: check_macro_code_does_not_contain_regexp_pattern\n regexp_pattern: .*[i][f][n][u][l][l].*\n ```\n\n \"\"\"\n assert (\n re.compile(regexp_pattern.strip(), flags=re.DOTALL).match(macro.macro_sql)\n is None\n ), f\"Macro `{macro.name}` contains a banned string: `{regexp_pattern.strip()}`.\"\n
# Only \"common\" macros need to have a populated description\nmanifest_checks:\n - name: check_macro_description_populated\n include: ^macros/common\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_description_populated(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macros must have a populated description.\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_description_populated\n ```\n ```yaml\n # Only \"common\" macros need to have a populated description\n manifest_checks:\n - name: check_macro_description_populated\n include: ^macros/common\n ```\n\n \"\"\"\n assert (\n len(macro.description.strip()) > 4\n ), f\"Macro `{macro.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_max_number_of_lines(\n macro: \"Macros\",\n max_number_of_lines: int = 50,\n **kwargs,\n) -> None:\n \"\"\"Macros may not have more than the specified number of lines.\n\n Parameters:\n macro (Macros): The Macros object to check.\n max_number_of_lines (int): The maximum number of permitted lines.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_max_number_of_lines\n ```\n ```yaml\n manifest_checks:\n - name: check_macro_max_number_of_lines\n max_number_of_lines: 100\n ```\n\n \"\"\"\n actual_number_of_lines = macro.macro_sql.count(\"\\n\") + 1\n\n assert (\n actual_number_of_lines <= max_number_of_lines\n ), f\"Macro `{macro.name}` has {actual_number_of_lines} lines, this is more than the maximum permitted number of lines ({max_number_of_lines}).\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_name_matches_file_name(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macros names must be the same as the file they are contained in.\n\n Generic tests are also macros, however to document these tests the \"name\" value must be preceded with \"test_\".\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_name_matches_file_name\n ```\n\n \"\"\"\n if macro.name.startswith(\"test_\"):\n assert (\n macro.name[5:] == macro.original_file_path.split(\"/\")[-1].split(\".\")[0]\n ), f\"Macro `{macro.unique_id}` is not in a file named `{macro.name[5:]}.sql`.\"\n else:\n assert (\n macro.name == macro.original_file_path.split(\"/\")[-1].split(\".\")[0]\n ), f\"Macro `{macro.name}` is not in a file of the same name.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_property_file_location(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macro properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/5-the-rest-of-the-project#how-we-use-the-other-folders).\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_property_file_location\n ```\n\n \"\"\"\n expected_substr = \"_\".join(macro.original_file_path[6:].split(\"/\")[:-1])\n properties_yml_name = macro.patch_path.split(\"/\")[-1]\n\n if macro.original_file_path.startswith(\n \"tests/\",\n ): # Do not check generic tests (which are also macros)\n pass\n elif expected_substr == \"\": # i.e. macro in ./macros\n assert (\n properties_yml_name == \"_macros.yml\"\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) should be `_macros.yml`.\"\n else:\n assert properties_yml_name.startswith(\n \"_\",\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) does not start with an underscore.\"\n assert (\n expected_substr in properties_yml_name\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substr}`).\"\n assert properties_yml_name.endswith(\n \"__macros.yml\",\n ), f\"The properties file for `{macro.name.name}` (`{properties_yml_name}`) does not end with `__macros.yml`.\"\n
Enforce that the name of the dbt project matches a supplied regex. Generally used to enforce that project names conform to something like company_<DOMAIN>.
Parameters:
Name Type Description Default manifest_objDbtBouncerManifest
The manifest object.
required project_name_patternstr
Regex pattern to match the project name.
required
Other Parameters (passed via config file):
Name Type Description severityOptional[Literal['error', 'warn']]
Source code in src/dbt_bouncer/checks/manifest/check_metadata.py
def check_project_name(\n manifest_obj: \"DbtBouncerManifest\",\n project_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Enforce that the name of the dbt project matches a supplied regex. Generally used to enforce that project names conform to something like `company_<DOMAIN>`.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The manifest object.\n project_name_pattern (str): Regex pattern to match the project name.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_project_name\n project_name_pattern: ^awesome_company_\n ```\n\n \"\"\"\n assert (\n re.compile(project_name_pattern.strip()).match(\n manifest_obj.manifest.metadata.project_name,\n )\n is not None\n ), f\"Project name (`{manifest_obj.manifest.metadata.project_name}`) does not conform to the supplied regex `({project_name_pattern.strip()})`.\"\n
Models must have the specified access attribute. Requires dbt 1.7+.
Parameters:
Name Type Description Default accessLiteral['private', 'protected', 'public']
The access level to check for.
required modelDbtBouncerModel
The DbtBouncerModel object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the model path. Model paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the model path. Only model paths that match the pattern will be checked.
severityOptional[Literal['error', 'warn']]
Severity level of the check. Default: error.
Example(s):
manifest_checks:\n # Align with dbt best practices that marts should be `public`, everything else should be `protected`\n - name: check_model_access\n access: protected\n include: ^models/intermediate\n - name: check_model_access\n access: public\n include: ^models/marts\n - name: check_model_access\n access: protected\n include: ^models/staging\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_access(\n access: str,\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"Models must have the specified access attribute. Requires dbt 1.7+.\n\n Parameters:\n access (Literal[\"private\", \"protected\", \"public\"]): The access level to check for.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Align with dbt best practices that marts should be `public`, everything else should be `protected`\n - name: check_model_access\n access: protected\n include: ^models/intermediate\n - name: check_model_access\n access: public\n include: ^models/marts\n - name: check_model_access\n access: protected\n include: ^models/staging\n ```\n\n \"\"\"\n assert (\n model.access.value == access\n ), f\"`{model.name}` has `{model.access.value}` access, it should have access `{access}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_contract_enforced_for_public_model(\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"Public models must have contracts enforced.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_contract_enforced_for_public_model\n ```\n\n \"\"\"\n if model.access.value == \"public\":\n assert (\n model.contract.enforced is True\n ), f\"`{model.name}` is a public model but does not have contracts enforced.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_description_populated(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Models must have a populated description.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_description_populated\n ```\n\n \"\"\"\n assert (\n len(model.description.strip()) > 4\n ), f\"`{model.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_documentation_coverage(\n models: List[\"DbtBouncerModel\"],\n min_model_documentation_coverage_pct: float = 100,\n **kwargs,\n) -> None:\n \"\"\"Set the minimum percentage of models that have a populated description.\n\n Parameters:\n min_model_documentation_coverage_pct (float): The minimum percentage of models that must have a populated description.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_description_populated\n min_model_documentation_coverage_pct: 90\n ```\n\n \"\"\"\n num_models = len(models)\n models_with_description = []\n for model in models:\n if len(model.description.strip()) > 4:\n models_with_description.append(model.unique_id)\n\n num_models_with_descriptions = len(models_with_description)\n model_description_coverage_pct = (num_models_with_descriptions / num_models) * 100\n\n assert (\n model_description_coverage_pct >= min_model_documentation_coverage_pct\n ), f\"Only {model_description_coverage_pct}% of models have a populated description, this is less than the permitted minimum of {min_model_documentation_coverage_pct}%.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_documented_in_same_directory(\n model: \"DbtBouncerModel\", **kwargs\n) -> None:\n \"\"\"Models must be documented in the same directory where they are defined (i.e. `.yml` and `.sql` files are in the same directory).\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_documented_in_same_directory\n ```\n\n \"\"\"\n model_doc_dir = model.patch_path[model.patch_path.find(\"models\") :].split(\"/\")[:-1]\n model_sql_dir = model.original_file_path.split(\"/\")[:-1]\n\n assert (\n model_doc_dir == model_sql_dir\n ), f\"`{model.name}` is documented in a different directory to the `.sql` file: `{'/'.join(model_doc_dir)}` vs `{'/'.join(model_sql_dir)}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_code_does_not_contain_regexp_pattern(\n model: \"DbtBouncerModel\",\n regexp_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"The raw code for a model must not match the specified regexp pattern.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n regexp_pattern (str): The regexp pattern that should not be matched by the model code.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Prefer `coalesce` over `ifnull`: https://docs.sqlfluff.com/en/stable/rules.html#sqlfluff.rules.sphinx.Rule_CV02\n - name: check_model_code_does_not_contain_regexp_pattern\n regexp_pattern: .*[i][f][n][u][l][l].*\n ```\n\n \"\"\"\n assert (\n re.compile(regexp_pattern.strip(), flags=re.DOTALL).match(model.raw_code)\n is None\n ), f\"`{model.name}` contains a banned string: `{regexp_pattern.strip()}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_depends_on_multiple_sources(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Models cannot reference more than one source.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_depends_on_multiple_sources\n ```\n\n \"\"\"\n num_reffed_sources = sum(\n x.split(\".\")[0] == \"source\" for x in model.depends_on.nodes\n )\n assert num_reffed_sources <= 1, f\"`{model.name}` references more than one source.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_contracts_enforced(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Model must have contracts enforced.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_contracts_enforced\n include: ^models/marts\n ```\n\n \"\"\"\n assert (\n model.contract.enforced is True\n ), f\"`{model.name}` does not have contracts enforced.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_meta_keys(\n keys: NestedDict,\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"The `meta` config for models must have the specified keys.\n\n Parameters:\n keys (NestedDict): A list (that may contain sub-lists) of required keys.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_meta_keys\n keys:\n - maturity\n - owner\n ```\n\n \"\"\"\n missing_keys = find_missing_meta_keys(\n meta_config=model.meta,\n required_keys=keys,\n )\n assert (\n missing_keys == []\n ), f\"`{model.name}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_no_upstream_dependencies(\n model: \"DbtBouncerModel\", **kwargs\n) -> None:\n \"\"\"Identify if models have no upstream dependencies as this likely indicates hard-coded tables references.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_no_upstream_dependencies\n ```\n\n \"\"\"\n assert (\n len(model.depends_on.nodes) > 0\n ), f\"`{model.name}` has no upstream dependencies, this likely indicates hard-coded tables references.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_tags(model: \"DbtBouncerModel\", tags: List[str], **kwargs) -> None:\n \"\"\"Models must have the specified tags.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n tags (List[str]): List of tags to check for.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_tags\n tags:\n - tag_1\n - tag_2\n ```\n\n \"\"\"\n missing_tags = [tag for tag in tags if tag not in model.tags]\n assert not missing_tags, f\"`{model.name}` is missing required tags: {missing_tags}.\"\n
manifest_checks:\n # Example of allowing a custom uniqueness test\n - name: check_model_has_unique_test\n accepted_uniqueness_tests:\n - expect_compound_columns_to_be_unique\n - my_custom_uniqueness_test\n - unique\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_unique_test(\n model: \"DbtBouncerModel\",\n tests: \"DbtBouncerTest\",\n accepted_uniqueness_tests: List[str] = (\n [ # noqa: B006\n \"expect_compound_columns_to_be_unique\",\n \"dbt_utils.unique_combination_of_columns\",\n \"unique\",\n ]\n ),\n **kwargs,\n) -> None:\n \"\"\"Models must have a test for uniqueness of a column.\n\n Parameters:\n accepted_uniqueness_tests (Optional[List[str]]): List of tests that are accepted as uniqueness tests.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_unique_test\n include: ^models/marts\n ```\n ```yaml\n manifest_checks:\n # Example of allowing a custom uniqueness test\n - name: check_model_has_unique_test\n accepted_uniqueness_tests:\n - expect_compound_columns_to_be_unique\n - my_custom_uniqueness_test\n - unique\n ```\n\n \"\"\"\n num_unique_tests = sum(\n test.attached_node == model.unique_id\n and test.test_metadata.name in accepted_uniqueness_tests\n for test in tests\n )\n assert (\n num_unique_tests >= 1\n ), f\"`{model.name}` does not have a test for uniqueness of a column.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_unit_tests(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n unit_tests: List[\"UnitTests\"],\n min_number_of_unit_tests: int = 1,\n **kwargs,\n) -> None:\n \"\"\"Models must have more than the specified number of unit tests.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n min_number_of_unit_tests (Optional[int]): The minimum number of unit tests that a model must have.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n unit_tests (List[UnitTests]): List of UnitTests objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_unit_tests\n include: ^models/marts\n ```\n ```yaml\n manifest_checks:\n - name: check_model_has_unit_tests\n min_number_of_unit_tests: 2\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n num_unit_tests = len(\n [\n t.unique_id\n for t in unit_tests\n if t.depends_on.nodes[0] == model.unique_id\n ],\n )\n assert (\n num_unit_tests >= min_number_of_unit_tests\n ), f\"`{model.name}` has {num_unit_tests} unit tests, this is less than the minimum of {min_number_of_unit_tests}.\"\n else:\n logging.warning(\n \"The `check_model_has_unit_tests` check is only supported for dbt 1.8.0 and above.\",\n )\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_chained_views(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n materializations_to_include: List[str] = [\"ephemeral\", \"view\"], # noqa: B006\n max_chained_views: int = 3,\n **kwargs,\n) -> None:\n \"\"\"Models cannot have more than the specified number of upstream dependents that are not tables.\n\n Parameters:\n materializations_to_include (Optional[List[str]]): List of materializations to include in the check.\n max_chained_views (Optional[int]): The maximum number of upstream dependents that are not tables.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_chained_views\n ```\n ```yaml\n manifest_checks:\n - name: check_model_max_chained_views\n materializations_to_include:\n - ephemeral\n - my_custom_materialization\n - view\n max_chained_views: 5\n ```\n\n \"\"\"\n\n def return_upstream_view_models(\n materializations,\n max_chained_views,\n models,\n model_unique_ids_to_check,\n package_name,\n depth=0,\n ):\n \"\"\"Recursive function to return model unique_id's of upstream models that are views. Depth of recursion can be specified. If no models meet the criteria then an empty list is returned.\n\n Returns\n -\n List[str]: List of model unique_id's of upstream models that are views.\n\n \"\"\"\n if depth == max_chained_views or model_unique_ids_to_check == []:\n return model_unique_ids_to_check\n\n relevant_upstream_models = []\n for model in model_unique_ids_to_check:\n upstream_nodes = list(\n next(m2 for m2 in models if m2.unique_id == model).depends_on.nodes,\n )\n if upstream_nodes != []:\n upstream_models = [\n m\n for m in upstream_nodes\n if m.split(\".\")[0] == \"model\" and m.split(\".\")[1] == package_name\n ]\n for i in upstream_models:\n if (\n next(m for m in models if m.unique_id == i).config.materialized\n in materializations\n ):\n relevant_upstream_models.append(i)\n\n depth += 1\n return return_upstream_view_models(\n materializations=materializations,\n max_chained_views=max_chained_views,\n models=models,\n model_unique_ids_to_check=relevant_upstream_models,\n package_name=package_name,\n depth=depth,\n )\n\n assert (\n len(\n return_upstream_view_models(\n materializations=materializations_to_include,\n max_chained_views=max_chained_views,\n models=models,\n model_unique_ids_to_check=[model.unique_id],\n package_name=manifest_obj.manifest.metadata.project_name,\n ),\n )\n == 0\n ), f\"`{model.name}` has more than {max_chained_views} upstream dependents that are not tables.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_fanout(\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n max_downstream_models: int = 3,\n **kwargs,\n) -> None:\n \"\"\"Models cannot have more than the specified number of downstream models.\n\n Parameters:\n max_downstream_models (Optional[int]): The maximum number of permitted downstream models.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_fanout\n max_downstream_models: 2\n ```\n\n \"\"\"\n num_downstream_models = sum(model.unique_id in m.depends_on.nodes for m in models)\n\n assert (\n num_downstream_models <= max_downstream_models\n ), f\"`{model.name}` has {num_downstream_models} downstream models, which is more than the permitted maximum of {max_downstream_models}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_number_of_lines(\n model: \"DbtBouncerModel\",\n max_number_of_lines: int = 100,\n **kwargs,\n) -> None:\n \"\"\"Models may not have more than the specified number of lines.\n\n Parameters:\n max_number_of_lines (int): The maximum number of permitted lines.\n\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_number_of_lines\n ```\n ```yaml\n manifest_checks:\n - name: check_model_max_number_of_lines\n max_number_of_lines: 150\n ```\n\n \"\"\"\n actual_number_of_lines = model.raw_code.count(\"\\n\") + 1\n\n assert (\n actual_number_of_lines <= max_number_of_lines\n ), f\"`{model.name}` has {actual_number_of_lines} lines, this is more than the maximum permitted number of lines ({max_number_of_lines}).\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_upstream_dependencies(\n model: \"DbtBouncerModel\",\n max_upstream_macros: int = 5,\n max_upstream_models: int = 5,\n max_upstream_sources: int = 1,\n **kwargs,\n) -> None:\n \"\"\"Limit the number of upstream dependencies a model has.\n\n Parameters:\n max_upstream_macros (Optional[int]): The maximum number of permitted upstream macros.\n max_upstream_models (Optional[int]): The maximum number of permitted upstream models.\n max_upstream_sources (Optional[int]): The maximum number of permitted upstream sources.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_upstream_dependencies\n max_upstream_models: 3\n ```\n\n \"\"\"\n num_upstream_macros = len(list(model.depends_on.macros))\n num_upstream_models = len(\n [m for m in model.depends_on.nodes if m.split(\".\")[0] == \"model\"],\n )\n num_upstream_sources = len(\n [m for m in model.depends_on.nodes if m.split(\".\")[0] == \"source\"],\n )\n\n assert (\n num_upstream_macros <= max_upstream_macros\n ), f\"`{model.name}` has {num_upstream_macros} upstream macros, which is more than the permitted maximum of {max_upstream_macros}.\"\n assert (\n num_upstream_models <= max_upstream_models\n ), f\"`{model.name}` has {num_upstream_models} upstream models, which is more than the permitted maximum of {max_upstream_models}.\"\n assert (\n num_upstream_sources <= max_upstream_sources\n ), f\"`{model.name}` has {num_upstream_sources} upstream sources, which is more than the permitted maximum of {max_upstream_sources}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_names(\n model: \"DbtBouncerModel\",\n model_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Models must have a name that matches the supplied regex.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n model_name_pattern (str): Regexp the model name must match.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_names\n include: ^models/intermediate\n model_name_pattern: ^int_\n - name: check_model_names\n include: ^models/staging\n model_name_pattern: ^stg_\n ```\n\n \"\"\"\n assert (\n re.compile(model_name_pattern.strip()).match(model.name) is not None\n ), f\"`{model.name}` does not match the supplied regex `{model_name_pattern.strip()})`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_property_file_location(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Model properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/1-guide-overview).\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_property_file_location\n ```\n\n \"\"\"\n expected_substr = (\n \"_\".join(model.original_file_path.split(\"/\")[1:-1])\n .replace(\"staging\", \"stg\")\n .replace(\"intermediate\", \"int\")\n .replace(\"marts\", \"\")\n )\n properties_yml_name = model.patch_path.split(\"/\")[-1]\n\n assert properties_yml_name.startswith(\n \"_\",\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not start with an underscore.\"\n assert (\n expected_substr in properties_yml_name\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substr}`).\"\n assert properties_yml_name.endswith(\n \"__models.yml\",\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not end with `__models.yml`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_test_coverage(\n models: List[\"DbtBouncerModel\"],\n tests: List[\"DbtBouncerTest\"],\n min_model_test_coverage_pct: float = 100,\n **kwargs,\n) -> None:\n \"\"\"Set the minimum percentage of models that have at least one test.\n\n Parameters:\n min_model_test_coverage_pct (float): The minimum percentage of models that must have at least one test.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_test_coverage\n min_model_test_coverage_pct: 90\n ```\n\n \"\"\"\n num_models = len(models)\n models_with_tests = []\n for model in models:\n for test in tests:\n if model.unique_id in test.depends_on.nodes:\n models_with_tests.append(model.unique_id)\n num_models_with_tests = len(set(models_with_tests))\n model_test_coverage_pct = (num_models_with_tests / num_models) * 100\n\n assert (\n model_test_coverage_pct >= min_model_test_coverage_pct\n ), f\"Only {model_test_coverage_pct}% of models have at least one test, this is less than the permitted minimum of {min_model_test_coverage_pct}%.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_description_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated description.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_description_populated\n ```\n\n \"\"\"\n assert (\n len(source.description.strip()) > 4\n ), f\"`{source.source_name}.{source.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_freshness_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated freshness.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_freshness_populated\n ```\n\n \"\"\"\n assert (\n source.freshness.error_after.count is not None\n and source.freshness.error_after.period is not None\n ) or (\n source.freshness.warn_after.count is not None\n and source.freshness.warn_after.period is not None\n ), f\"`{source.source_name}.{source.name}` does not have a populated freshness.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_has_meta_keys(\n keys: \"NestedDict\",\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"The `meta` config for sources must have the specified keys.\n\n Parameters:\n keys (NestedDict): A list (that may contain sub-lists) of required keys.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_has_meta_keys\n keys:\n - contact:\n - email\n - slack\n - owner\n ```\n\n \"\"\"\n missing_keys = find_missing_meta_keys(\n meta_config=source.meta,\n required_keys=keys,\n )\n assert (\n missing_keys == []\n ), f\"`{source.source_name}.{source.name}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_has_tags(\n source: \"DbtBouncerSource\",\n tags: List[str],\n **kwargs,\n) -> None:\n \"\"\"Sources must have the specified tags.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n tags (List[str]): List of tags to check for.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_has_tags\n tags:\n - tag_1\n - tag_2\n ```\n\n \"\"\"\n missing_tags = [tag for tag in tags if tag not in source.tags]\n assert (\n not missing_tags\n ), f\"`{source.source_name}.{source.name}` is missing required tags: {missing_tags}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_loader_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated loader.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_loader_populated\n ```\n\n \"\"\"\n assert (\n source.loader != \"\"\n ), f\"`{source.source_name}.{source.name}` does not have a populated loader.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_names(\n source: \"DbtBouncerSource\",\n source_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Sources must have a name that matches the supplied regex.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n source_name_pattern (str): Regexp the source name must match.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_names\n source_name_pattern: >\n ^[a-z0-9_]*$\n ```\n\n \"\"\"\n assert (\n re.compile(source_name_pattern.strip()).match(source.name) is not None\n ), f\"`{source.source_name}.{source.name}` does not match the supplied regex `({source_name_pattern.strip()})`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_not_orphaned(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Sources must be referenced in at least one model.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_not_orphaned\n ```\n\n \"\"\"\n num_refs = sum(source.unique_id in model.depends_on.nodes for model in models)\n assert (\n num_refs >= 1\n ), f\"Source `{source.source_name}.{source.name}` is orphaned, i.e. not referenced by any model.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_property_file_location(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Source properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/1-guide-overview).\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_property_file_location\n ```\n\n \"\"\"\n path_cleaned = source.original_file_path.replace(\"models/staging\", \"\")\n expected_substring = \"_\".join(path_cleaned.split(\"/\")[:-1])\n\n assert path_cleaned.split(\n \"/\",\n )[\n -1\n ].startswith(\n \"_\",\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not start with an underscore.\"\n assert (\n expected_substring in path_cleaned\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not contain the expected substring (`{expected_substring}`).\"\n assert path_cleaned.split(\n \"/\",\n )[\n -1\n ].endswith(\n \"__sources.yml\",\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not end with `__sources.yml`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_used_by_models_in_same_directory(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Sources can only be referenced by models that are located in the same directory where the source is defined.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_used_by_models_in_same_directory\n ```\n\n \"\"\"\n reffed_models_not_in_same_dir = []\n for model in models:\n if (\n source.unique_id in model.depends_on.nodes\n and model.original_file_path.split(\"/\")[:-1]\n != source.original_file_path.split(\"/\")[:-1]\n ):\n reffed_models_not_in_same_dir.append(model.unique_id.split(\".\")[0])\n\n assert (\n len(reffed_models_not_in_same_dir) == 0\n ), f\"Source `{source.source_name}.{source.name}` is referenced by models defined in a different directory: {reffed_models_not_in_same_dir}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_used_by_only_one_model(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Each source can be referenced by a maximum of one model.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_used_by_only_one_model\n ```\n\n \"\"\"\n num_refs = sum(source.unique_id in model.depends_on.nodes for model in models)\n assert (\n num_refs <= 1\n ), f\"Source `{source.source_name}.{source.name}` is referenced by more than one model.\"\n
"},{"location":"checks/manifest/check_unit_tests/","title":"Manifest Checks: Unit Tests","text":"
Note
The below checks require manifest.json to be present.
A list of formats that are allowed to be used for expect input in a unit test.
['csv', 'dict', 'sql']unit_testUnitTests
The UnitTests object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_unit_tests.py
def check_unit_test_expect_format(\n manifest_obj: \"DbtBouncerManifest\",\n unit_test: \"UnitTests\",\n permitted_formats: List[Literal[\"csv\", \"dict\", \"sql\"]] = [\"csv\", \"dict\", \"sql\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Unit tests can only use the specified formats.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n permitted_formats (Optional[List[Literal[\"csv\", \"dict\", \"sql\"]]]): A list of formats that are allowed to be used for `expect` input in a unit test.\n unit_test (UnitTests): The UnitTests object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_unit_test_expect_format\n permitted_formats:\n - csv\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n assert (\n unit_test.expect.format.value in permitted_formats\n ), f\"Unit test `{unit_test.name}` has an `expect` format that is not permitted. Permitted formats are: {permitted_formats}.\"\n else:\n logging.warning(\n \"The `check_unit_test_expect_format` check is only supported for dbt 1.8.0 and above.\",\n )\n
A list of formats that are allowed to be used for expect input in a unit test.
['csv', 'dict', 'sql']unit_testUnitTests
The UnitTests object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_unit_tests.py
def check_unit_test_given_formats(\n manifest_obj: \"DbtBouncerManifest\",\n unit_test: \"UnitTests\",\n permitted_formats: List[Literal[\"csv\", \"dict\", \"sql\"]] = [\"csv\", \"dict\", \"sql\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Unit tests can only use the specified formats.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n permitted_formats (Optional[List[Literal[\"csv\", \"dict\", \"sql\"]]]): A list of formats that are allowed to be used for `expect` input in a unit test.\n unit_test (UnitTests): The UnitTests object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_unit_test_given_formats\n permitted_formats:\n - csv\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n given_formats = [i.format.value for i in unit_test.given]\n assert all(\n e in permitted_formats for e in given_formats\n ), f\"Unit test `{unit_test.name}` has given formats which are not permitted. Permitted formats are: {permitted_formats}.\"\n else:\n logging.warning(\n \"The `check_unit_test_given_formats` check is only supported for dbt 1.8.0 and above.\",\n )\n
Source code in src/dbt_bouncer/checks/run_results/check_run_results.py
def check_run_results_max_gigabytes_billed(\n max_gigabytes_billed: float,\n run_result: \"DbtBouncerRunResult\",\n **kwargs,\n) -> None:\n \"\"\"Each result can have a maximum number of gigabytes billed.\n\n !!! note\n\n Note that this check only works for the `dbt-bigquery` adapter.\n\n Parameters:\n max_gigabytes_billed (float): The maximum number of gigabytes billed.\n run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Raises: # noqa:DOC502\n KeyError: If the `dbt-bigquery` adapter is not used.\n\n Example(s):\n ```yaml\n run_results_checks:\n - name: check_run_results_max_gigabytes_billed\n max_gigabytes_billed: 100\n ```\n\n \"\"\"\n try:\n gigabytes_billed = run_result.adapter_response[\"bytes_billed\"] / (1000**3)\n except KeyError as e:\n raise RuntimeError( # noqa: DOC501\n \"`bytes_billed` not found in adapter response. Are you using the `dbt-bigquery` adapter?\",\n ) from e\n\n assert (\n gigabytes_billed < max_gigabytes_billed\n ), f\"`{run_result.unique_id.split('.')[-2]}` results in ({gigabytes_billed} billed bytes, this is greater than permitted ({max_gigabytes_billed}).\"\n
run_results_checks:\n - name: check_run_results_max_execution_time\n include: ^models/staging # Not a good idea, here for demonstration purposes only\n max_execution_time_seconds: 10\n
Source code in src/dbt_bouncer/checks/run_results/check_run_results.py
def check_run_results_max_execution_time(\n max_execution_time_seconds: float,\n run_result: \"DbtBouncerRunResult\",\n **kwargs,\n) -> None:\n \"\"\"Each result can take a maximum duration (seconds).\n\n Parameters:\n max_execution_time_seconds (float): The maximum execution time (seconds) allowed for a node.\n run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n run_results_checks:\n - name: check_run_results_max_execution_time\n max_execution_time_seconds: 60\n ```\n ```yaml\n run_results_checks:\n - name: check_run_results_max_execution_time\n include: ^models/staging # Not a good idea, here for demonstration purposes only\n max_execution_time_seconds: 10\n ```\n\n \"\"\"\n assert (\n run_result.execution_time <= max_execution_time_seconds\n ), f\"`{run_result.unique_id.split('.')[-1]}` has an execution time ({run_result.execution_time} greater than permitted ({max_execution_time_seconds}s).\"\n
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Home","text":""},{"location":"#welcome-to-dbt-bouncer","title":"Welcome to dbt-bouncer","text":"
dbt-bouncer is an open-source tool that allows you to configure and enforce conventions for your dbt project. The conventions are run against dbt's artifact files (think ./target/manifest.json) resulting in speedy tests. Conventions can be specified in a .yml file, allowing maximum customisation to the conventions you wish to follow (or create \ud83d\ude00).
Check: A check is a rule run against a dbt artifact.
Config file: A .yml file that specifies which checks to run along with any parameters.
dbt artifacts directory: The directory that contains the dbt artifacts (manifest.json, etc.), generally this is ./target.
"},{"location":"#aims","title":"Aims","text":"
dbt-bouncer aims to:
Provide a 100% configurable way to enforce conventions in a dbt project.
Be as fast as possible, running checks against dbt artifacts.
Be as easy as possible to use, with a simple config file written in YML.
Be as flexible as possible, allowing checks to be written in python.
Provide immediate feedback when run as part of a CI pipeline.
"},{"location":"#about","title":"About","text":"
dbt-bouncer is free software, released under the MIT license. It originated at Xebia Data in Amsterdam, Netherlands. Source code is available on GitHub.
All contributions, in the form of bug reports, pull requests, feedback or discussion are welcome. See the contributing guide for more information.
"},{"location":"CONTRIBUTING/","title":"Contributing to dbt-bouncer","text":"
dbt-bouncer is open source software. Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
"},{"location":"CONTRIBUTING/#about-this-document","title":"About this document","text":"
There are many ways to contribute to the ongoing development of dbt-bouncer, such as by participating in discussions and issues.
The rest of this document serves as a more granular guide for contributing code changes to dbt-bouncer (this repository). It is not intended as a guide for using dbt-bouncer, and some pieces assume a level of familiarity with Python development (virtualenvs, Poetry, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line.
If you get stuck, we're happy to help! Just open an issue or draft PR and we'll do our best to help out.
Branches: All pull requests from community contributors should target the main branch (default).
"},{"location":"CONTRIBUTING/#getting-the-code","title":"Getting the code","text":""},{"location":"CONTRIBUTING/#installing-git","title":"Installing git","text":"
You will need git in order to download and modify the dbt-bouncer source code. On macOS, the best way to download git is to just install Xcode.
You can contribute to dbt-bouncer by forking the dbt-bouncer repository. For a detailed overview on forking, check out the GitHub docs on forking. In short, you will need to:
Fork the dbt-bouncer repository.
Clone your fork locally.
Check out a new branch for your proposed changes.
Push changes to your fork.
Open a pull request against godatadriven/dbt-bouncer from your forked repository.
"},{"location":"CONTRIBUTING/#setting-up-an-environment","title":"Setting up an environment","text":"
There are some tools that will be helpful to you in developing locally. While this is the list relevant for dbt-bouncer development, many of these tools are used commonly across open-source python projects.
These are the tools used in dbt-bouncer development and testing:
click to create our CLI interface.
GitHub Actions for automating tests and checks, once a PR is pushed to the dbt-bouncer repository.
make to run multiple setup or test steps in combination.
mypy for static type checking.
Poetry to manage our python virtual environment.
pre-commit to easily run those checks.
Pydantic to validate our configuration file.
pytest to define, discover, and run tests.
Ruff to lint and format python code.
A deep understanding of these tools in not required to effectively contribute to dbt-bouncer, but we recommend checking out the attached documentation if you're interested in learning more about each one.
We strongly recommend using virtual environments when developing code in dbt-bouncer. We recommend creating this virtualenv in the root of the dbt-bouncer repository. To create a new virtualenv, run:
poetry shell\n
This will create a new Python virtual environment.
Set required environment variables by copying .env.example to .env and updating the values.
"},{"location":"CONTRIBUTING/#running-dbt-bouncer-in-development","title":"Running dbt-bouncer in development","text":""},{"location":"CONTRIBUTING/#installation","title":"Installation","text":"
First make sure that you set up your virtualenv as described in Setting up an environment. Next, install dbt-bouncer, its dependencies and pre-commit:
poetry install\npoetry run pre-commit install\n
When installed in this way, any changes you make to your local copy of the source code will be reflected immediately in your next dbt-bouncer run.
With your virtualenv activated, the dbt-bouncer script should point back to the source code you've cloned on your machine. You can verify this by running which dbt-bouncer. This command should show you a path to an executable in your virtualenv. You can run dbt-bouncer using the provided example configuration file via:
poetry run dbt-bouncer --config-file dbt-bouncer-example.yml\n
Once you're able to manually test that your code change is working as expected, it's important to run existing automated tests, as well as adding some new ones. These tests will ensure that: - Your code changes do not unexpectedly break other established functionality - Your code changes can handle all known edge cases - The functionality you're adding will keep working in the future
Generating dbt artifacts: If you change the configuration of the dbt project located in dbt_project then you will need to re-generate the dbt artifacts used in testing. To do so, run:
pre-commit takes care of running all code-checks for formatting and linting. Run poetry run pre-commit install to install pre-commit in your local environment. Once this is done you can use the git pre-commit hooks to ensure proper formatting and linting.
Finally, you can also run a specific test or group of tests using pytest directly. With a virtualenv active and dev dependencies installed you can do things like:
# run all unit tests in a file\npoetry run pytest ./tests/unit/checks/catalog/test_columns.py\n\n# run a specific unit test\npoetry run pytest ./tests/unit/checks/catalog/test_columns.py::test_check_columns_are_documented_in_public_models\n
See pytest usage docs for an overview of useful command-line options.
"},{"location":"CONTRIBUTING/#assorted-development-tips","title":"Assorted development tips","text":"
Append # type: ignore to the end of a line if you need to disable mypy on that line, preferably with the specific rule to ignore such as # type: ignore[union-attr].
"},{"location":"CONTRIBUTING/#adding-a-new-check","title":"Adding a new check","text":"
To add a new check follow the below steps:
In ./src/dbt_bouncer/checks choose the appropriate directory for your check. For example, if your check only requires the manifest.json then use the manifest directory, if your check requires the catalog.json then use the catalog directory.
Within the chosen directory assess if a suitable file already exists. For example, if your check applies to a model then manifest/check_models.py is a suitable location.
Within the chosen file, add both a class and a function:
class: The class is a pydantic model defining the input arguments and must meet the following criteria:
Start with \"Check\".
Inherit from dbt_bouncer.conf_validator_base.BaseCheck.
Have a name attribute that is a string.
Not use description in a Field.
A default value provided for optional input arguments.
function: The function must meet the following criteria:
Be called after the snake case equivalent of the name attribute of the created class.
Accept **kwargs.
Have a doc string that includes a description of the check, a list of possible input parameters and at least one example.
A clear message in the event of a failure.
After the check is added, add the check to dbt-bouncer-example.yml and run dbt-bouncer --config-file dbt-bouncer-example.yml to ensure the check succeeds.
(Optional) If the dbt project located in ./dbt_project needs to be updated then do so and also run make build-artifacts to generate the new test artifacts.
Add at least one happy path and one unhappy path test to ./tests. The appropriate test file will be the one matching the directory of the check. For example, if the check is in ./src/dbt_bouncer/checks/catalog/check_columns.py then the test file will be ./tests/unit/checks/catalog/test_columns.py.
Run make test to ensure the tests pass.
Open a PR \ud83c\udf89!
"},{"location":"CONTRIBUTING/#submitting-a-pull-request","title":"Submitting a Pull Request","text":"
Code can be merged into the current development branch main by opening a pull request. If the proposal looks like it's on the right track, then a dbt-bouncer maintainer will review the PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. Once merged, your contribution will be available for the next release of dbt-bouncer.
Automated tests run via GitHub Actions. If you're a first-time contributor, all tests will require a maintainer to approve.
Once all tests are passing and your PR has been approved, a dbt-bouncer maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:
"},{"location":"cli/","title":"CLI","text":"
This page provides documentation for the dbt-bouncer CLI.
Raises: RuntimeError: If output file has an invalid extension.
Usage:
dbt-bouncer [OPTIONS]\n
Options:
--config-file PATH Location of the YML config file.\n --output-file PATH Location of the json file where check metadata will be\n saved.\n -v, --verbosity Verbosity.\n --version Show the version and exit.\n --help Show this message and exit.\n
dbt-bouncer requires a config file which determines what checks are run. The following options are available, in order of priority:
A file passed via the --config-file CLI flag.
A file named dbt-bouncer.yml in the current working directory.
A [tool.dbt-bouncer] section in pyproject.toml.
Here is an example config file in yaml:
# [Optional] Directory where the dbt artifacts exists, generally the `target` directory inside a dbt project. Defaults to `./target`.\ndbt_artifacts_dir: target\n\nmanifest_checks:\n - name: check_macro_name_matches_file_name\n - name: check_model_names\n include: ^models/staging\n model_name_pattern: ^stg_\n
And the same config in toml:
[tool.dbt-bouncer]\n# [Optional] Directory where the dbt artifacts exists, generally the `target` directory inside a dbt project. Defaults to `./target`.\ndbt_artifacts_dir = \"target\"\n\n[[tool.dbt-bouncer.manifest_checks]]\nname = \"check_macro_name_matches_file_name\"\n\n[[tool.dbt-bouncer.manifest_checks]]\nname = \"check_model_names\"\ninclude = \"^models/staging\"\nmodel_name_pattern = \"^stg_\"\n
For more example config files, see here.
"},{"location":"config_file/#common-arguments","title":"Common arguments","text":""},{"location":"config_file/#exclude-and-include","title":"Exclude and Include","text":"
Most (but not all) checks accept the following optional arguments:
exclude: Regexp to match which original file paths to exclude.
include: Regexp to match which original file paths to include.
Example per resource type:
Exposures: The original file path to the properties file where the source is defined, e.g. ^models/marts/finance will match exposures defined in ./models/marts/finance/_exposures.yml.
Macros: The original file path to the macro file, e.g. ^macros/system will match files like ./macros/system/generate_schema_name.sql.
Models: The original file path to the model file, e.g. ^marts will match files like ./models/marts/customers.sql.
Run results: The original file path to the file associated with the resource, e.g. ^seeds/finance will match seeds in ./seeds/finance, ^models/staging will match models and tests in ./models/staging.
Semantic models: The original file path to the properties file where the semantic model is defined, e.g. ^models/marts/finance will match semantic models defined in ./models/marts/finance/_finance__semantic_models.yml.
Sources: The original file path to the properties file where the source is defined, e.g. ^models/staging/crm will match sources defined in ./models/staging/crm/_crm__sources.yml.
Unit tests: The original file path to the properties file where the unit test is defined, e.g. ^models/staging/crm will match unit tests defined in ^staging/crm/_stg_crm__unit_tests.yml.
To determine if a check accepts these arguments view the Checks page.
Note
exclude and include can be specified at both the check level and the global level. Should both levels be specified, then the check level is applied. All the below examples result in the check_model_names check being run on all models in ./models/staging:
# Specify `include` at the check level only\nmanifest_checks:\n - name: check_model_names\n include: ^models/staging\n model_name_pattern: ^stg_\n
# Specify `include` at the check and global levels\ninclude: ^models/marts\nmanifest_checks:\n - name: check_model_names\n include: ^models/staging\n model_name_pattern: ^stg_\n
# Specify `include` at the global level only\ninclude: ^models/staging\nmanifest_checks:\n - name: check_model_names\n model_name_pattern: ^stg_\n
All checks accept a severity argument, valid values are:
error: If the check fails then dbt-bouncer will return a non-zero exit code.
warn: If the check fails then dbt-bouncer will return a non-zero exit code.
severity can also be specified globally, this is useful when applying dbt-bouncer to a pre-existing dbt project. It allows you to run dbt-bouncer, identify the checks that fail and address the failures in your own time without receiving non-zero exit codes:
# Specify `severity` at the global levels: all checks will have a `warn` severity, avoiding non-zero exit codes.\nseverity: warn\n\nmanifest_checks:\n - name: check_exposure_based_on_view\n ...\n
Note
severity can be specified at both the check level and the global level. Should both levels be specified, then the global level is applied.
# No `severity` specified: check will have an `error` severity.\nmanifest_checks:\n - name: check_exposure_based_on_view\n
# Specify `severity` at the check level only: check will have a `warn` severity.\nmanifest_checks:\n - name: check_exposure_based_on_view\n severity: warn\n
# Specify `severity` at the check and global levels: check will have a `warn` severity.\nseverity: warn\nmanifest_checks:\n - name: check_exposure_based_on_view\n severity: error\n
# Specify `severity` at the global level only: check will have a `warn` severity.\nseverity: warn\nmanifest_checks:\n - name: check_exposure_based_on_view\n
"},{"location":"faq/","title":"Frequently Asked Questions","text":""},{"location":"faq/#can-other-tools-perform-the-same-checks-as-dbt-bouncer","title":"Can other tools perform the same checks as dbt-bouncer?","text":"
There are several other tools that perform similar tasks as dbt-bouncer.
dbt-checkpoint: A collection of pre-commit hooks for dbt projects. Tests are written in python. Configuration is performed via .pre-commit-config.yaml. Provided the dbt artifacts have already been generated, dbt-checkpoint does not need access to the underlying database. The hooks execute when a new commit is made, as such dbt-checkpoint is designed to be run only as part of pre-commit.
dbt-project-evaluator: This is a dbt package from dbt Labs. Tests are written in .sql files using a combination of Jinja and SQL. Configuration is performed via dbt_project.yml and seed files (i.e. csv files). Requires a connection to underlying database. Designed to be run both in a CI pipeline and also during active development.
dbt-score: This is a python package installable via pip. A collection of tests that apply only to dbt models. Tests can be executed from the command line. Tests are written in python. Configuration is performed via a pyproject.toml file. Provided the dbt artifacts have already been generated, dbt-score does not need access to the underlying database. Designed to be run during development.
While the above tools inhabit the same space as dbt-bouncer they do not provide what we consider to be the optimum experience that dbt-bouncer provides:
Designed to run both locally and in a CI pipeline.
Configurable via a file format, YML, that dbt developers are already familiar with.
Does not require database access.
Can run tests against any of dbt's artifacts.
Allows tests to be written in python.
As such we consider dbt-bouncer to be the best tool to enforce conventions in a dbt project.
Tip
dbt-bouncer can perform all the tests currently included in dbt-checkpoint, dbt-project-evaluator and dbt-score. If you see an existing test that is not possible with dbt-bouncer, open an issue and we'll add it!
"},{"location":"faq/#how-to-set-up-dbt-bouncer-in-a-monorepo","title":"How to set up dbt-bouncer in a monorepo?","text":"
A monorepo may consist of one directory with a dbt project and other directories with unrelated code. It may be desired for dbt-bouncer to be configured from the root directory. Sample directory tree:
To ease configuration you can use exclude or include at the global level (see Config File for more details). For the above example dbt-bouncer.yml could be configured as:
dbt-bouncer can now be run from the root directory.
"},{"location":"getting_started/","title":"Getting Started","text":""},{"location":"getting_started/#how-to-run-dbt-bouncer","title":"How to run dbt-bouncer","text":"
Generate dbt artifacts by running a dbt command:
dbt parse to generate a manifest.json artifact.
dbt docs generate to generate a catalog.json artifact (necessary if you are using catalog checks).
dbt run (or any other command that implies it e.g. dbt build) to generate a run_results.json artifact (necessary if you are using run results checks).
Create a dbt-bouncer.yml config file, details here.
Run dbt-bouncer to validate that your conventions are being maintained.
Source code in src/dbt_bouncer/checks/catalog/check_catalog_sources.py
def check_source_columns_are_all_documented(\n catalog_source: \"DbtBouncerCatalogNode\",\n sources: List[\"DbtBouncerSource\"],\n **kwargs,\n) -> None:\n \"\"\"All columns in a source should be included in the source's properties file, i.e. `.yml` file.\n\n Parameters:\n catalog_source (DbtBouncerCatalogNode): The DbtBouncerCatalogNode object to check.\n sources (List[DbtBouncerSource]): List of DbtBouncerSource objects parsed from `catalog.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_source_columns_are_all_documented\n ```\n\n \"\"\"\n source = next(s for s in sources if s.unique_id == catalog_source.unique_id)\n undocumented_columns = [\n v.name\n for _, v in catalog_source.columns.items()\n if v.name not in source.columns\n ]\n assert not undocumented_columns, f\"`{catalog_source.unique_id}` has columns that are not included in the sources properties file: {undocumented_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_description_populated(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Columns must have a populated description.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_column_description_populated\n include: ^models/marts\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n non_complying_columns = []\n for _, v in catalog_node.columns.items():\n if (\n model.columns.get(v.name) is None\n or len(model.columns[v.name].description.strip()) <= 4\n ):\n non_complying_columns.append(v.name)\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that do not have a populated description: {non_complying_columns}\"\n
Columns with specified data types must comply to the specified regexp naming pattern.
Parameters:
Name Type Description Default catalog_nodeCatalogTable
The CatalogTable object to check.
required column_name_patternstr
(str): Regex pattern to match the model name.
required typesList[str]
List of data types to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the model path. Model paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the model path. Only model paths that match the pattern will be checked.
severityOptional[Literal['error', 'warn']]
Severity level of the check. Default: error.
Example(s):
catalog_checks:\n # DATE columns must end with \"_date\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: .*_date$\n types:\n - DATE\n
catalog_checks:\n # BOOLEAN columns must start with \"is_\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^is_.*\n types:\n - BOOLEAN\n
catalog_checks:\n # Columns of all types must consist of lowercase letters and underscores. Note that the specified types depend on the underlying database.\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^[a-z_]*$\n types:\n - BIGINT\n - BOOLEAN\n - DATE\n - DOUBLE\n - INTEGER\n - VARCHAR\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_name_complies_to_column_type(\n catalog_node: \"CatalogTable\",\n column_name_pattern: str,\n types: List[str],\n **kwargs,\n) -> None:\n \"\"\"Columns with specified data types must comply to the specified regexp naming pattern.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n column_name_pattern: (str): Regex pattern to match the model name.\n types (List[str]): List of data types to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n # DATE columns must end with \"_date\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: .*_date$\n types:\n - DATE\n ```\n ```yaml\n catalog_checks:\n # BOOLEAN columns must start with \"is_\"\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^is_.*\n types:\n - BOOLEAN\n ```\n ```yaml\n catalog_checks:\n # Columns of all types must consist of lowercase letters and underscores. Note that the specified types depend on the underlying database.\n - name: check_column_name_complies_to_column_type\n column_name_pattern: ^[a-z_]*$\n types:\n - BIGINT\n - BOOLEAN\n - DATE\n - DOUBLE\n - INTEGER\n - VARCHAR\n ```\n\n \"\"\"\n non_complying_columns = [\n v.name\n for _, v in catalog_node.columns.items()\n if v.type in types\n and re.compile(column_name_pattern.strip()).match(v.name) is None\n ]\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that don't comply with the specified regexp pattern (`{column_name_pattern}`): {non_complying_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_columns_are_all_documented(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"All columns in a model should be included in the model's properties file, i.e. `.yml` file.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_columns_are_all_documented\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n undocumented_columns = [\n v.name\n for _, v in catalog_node.columns.items()\n if v.name not in model.columns\n ]\n assert not undocumented_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that are not included in the models properties file: {undocumented_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_columns_are_documented_in_public_models(\n catalog_node: \"CatalogTable\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Columns should have a populated description in public models.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_columns_are_documented_in_public_models\n ```\n\n \"\"\"\n if catalog_node.unique_id.split(\".\")[0] == \"model\":\n model = next(m for m in models if m.unique_id == catalog_node.unique_id)\n non_complying_columns = []\n for _, v in catalog_node.columns.items():\n if model.access.value == \"public\":\n column_config = model.columns.get(v.name)\n if column_config is None or len(column_config.description.strip()) < 4:\n non_complying_columns.append(v.name)\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` is a public model but has columns that don't have a populated description: {non_complying_columns}\"\n
Source code in src/dbt_bouncer/checks/catalog/check_columns.py
def check_column_has_specified_test(\n catalog_node: \"CatalogTable\",\n column_name_pattern: str,\n test_name: str,\n tests: List[\"DbtBouncerTest\"],\n **kwargs,\n) -> None:\n \"\"\"Columns that match the specified regexp pattern must have a specified test.\n\n Parameters:\n catalog_node (CatalogTable): The CatalogTable object to check.\n column_name_pattern (str): Regex pattern to match the column name.\n test_name (str): Name of the test to check for.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n catalog_checks:\n - name: check_column_has_specified_test\n column_name_pattern: ^is_.*\n test_name: not_null\n ```\n\n \"\"\"\n columns_to_check = [\n v.name\n for _, v in catalog_node.columns.items()\n if re.compile(column_name_pattern.strip()).match(v.name) is not None\n ]\n relevant_tests = [\n t\n for t in tests\n if t.test_metadata.name == test_name\n and t.attached_node == catalog_node.unique_id\n ]\n non_complying_columns = [\n c\n for c in columns_to_check\n if f\"{catalog_node.unique_id}.{c}\"\n not in [f\"{t.attached_node}.{t.column_name}\" for t in relevant_tests]\n ]\n\n assert not non_complying_columns, f\"`{catalog_node.unique_id.split('.')[-1]}` has columns that should have a `{test_name}` test: {non_complying_columns}\"\n
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_exposures.py
def check_exposure_based_on_non_public_models(\n exposure: \"Exposures\",\n models: List[\"DbtBouncerModel\"],\n **kwargs,\n) -> None:\n \"\"\"Exposures should be based on public models only.\n\n Parameters:\n exposure (Exposures): The Exposures object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_non_public_models\n ```\n\n \"\"\"\n non_public_upstream_dependencies = []\n for model in exposure.depends_on.nodes:\n if (\n model.split(\".\")[0] == \"model\"\n and model.split(\".\")[1] == exposure.package_name\n ):\n model = next(m for m in models if m.unique_id == model)\n if model.access.value != \"public\":\n non_public_upstream_dependencies.append(model.name)\n\n assert not non_public_upstream_dependencies, f\"`{exposure.name}` is based on a model(s) that is not public: {non_public_upstream_dependencies}.\"\n
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_exposures.py
def check_exposure_based_on_view(\n exposure: \"Exposures\",\n models: List[\"DbtBouncerModel\"],\n materializations_to_include: List[str] = [\"ephemeral\", \"view\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Exposures should not be based on views.\n\n Parameters:\n exposure (Exposures): The Exposures object to check.\n materializations_to_include (Optional[List[str]]): List of materializations to include in the check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Exposure paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the exposure path (i.e the .yml file where the exposure is configured). Only exposure paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_view\n ```\n ```yaml\n manifest_checks:\n - name: check_exposure_based_on_view\n materializations_to_include:\n - ephemeral\n - my_custom_materialization\n - view\n ```\n\n \"\"\"\n non_table_upstream_dependencies = []\n for model in exposure.depends_on.nodes:\n if (\n model.split(\".\")[0] == \"model\"\n and model.split(\".\")[1] == exposure.package_name\n ):\n model = next(m for m in models if m.unique_id == model)\n if model.config.materialized in materializations_to_include:\n non_table_upstream_dependencies.append(model.name)\n\n assert not non_table_upstream_dependencies, f\"`{exposure.name}` is based on a model that is not a table: {non_table_upstream_dependencies}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_permitted_upstream_models(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n upstream_path_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Upstream models must have a path that matches the provided `upstream_path_pattern`.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The manifest object.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n upstream_path_pattern (str): Regexp pattern to match the upstream model(s) path.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_permitted_upstream_models\n include: ^models/staging\n upstream_path_pattern: $^\n - name: check_lineage_permitted_upstream_models\n include: ^models/intermediate\n upstream_path_pattern: ^models/staging|^models/intermediate\n - name: check_lineage_permitted_upstream_models\n include: ^models/marts\n upstream_path_pattern: ^models/staging|^models/intermediate\n ```\n\n \"\"\"\n upstream_models = [\n x\n for x in model.depends_on.nodes\n if x.split(\".\")[0] == \"model\"\n and x.split(\".\")[1] == manifest_obj.manifest.metadata.project_name\n ]\n not_permitted_upstream_models = [\n upstream_model\n for upstream_model in upstream_models\n if re.compile(upstream_path_pattern.strip()).match(\n next(m for m in models if m.unique_id == upstream_model).original_file_path,\n )\n is None\n ]\n assert not not_permitted_upstream_models, f\"`{model.name}` references upstream models that are not permitted: {[m.split('.')[-1] for m in not_permitted_upstream_models]}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_seed_cannot_be_used(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Seed cannot be referenced in models with a path that matches the specified `include` config.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_seed_cannot_be_used\n include: ^models/intermediate|^models/marts\n ```\n\n \"\"\"\n assert not [\n x for x in model.depends_on.nodes if x.split(\".\")[0] == \"seed\"\n ], f\"`{model.name}` references a seed even though this is not permitted.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_lineage.py
def check_lineage_source_cannot_be_used(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Sources cannot be referenced in models with a path that matches the specified `include` config.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_lineage_source_cannot_be_used\n include: ^models/intermediate|^models/marts\n ```\n\n \"\"\"\n assert not [\n x for x in model.depends_on.nodes if x.split(\".\")[0] == \"source\"\n ], f\"`{model.name}` references a source even though this is not permitted.\"\n
# Only \"common\" macros need to have their arguments populated\nmanifest_checks:\n - name: check_macro_arguments_description_populated\n include: ^macros/common\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_arguments_description_populated(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macro arguments must have a populated description.\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_arguments_description_populated\n ```\n ```yaml\n # Only \"common\" macros need to have their arguments populated\n manifest_checks:\n - name: check_macro_arguments_description_populated\n include: ^macros/common\n ```\n\n \"\"\"\n environment = jinja2.Environment(autoescape=True, extensions=[TagExtension])\n ast = environment.parse(macro.macro_sql)\n\n # Assume macro is a \"true\" macro, if not see if it's a generic test\n try:\n macro_arguments = [a.name for a in ast.body[0].args] # type: ignore[attr-defined]\n except AttributeError:\n test_macro = next(\n x\n for x in ast.body\n if not isinstance(x.nodes[0], jinja2.nodes.Call) # type: ignore[attr-defined]\n )\n macro_arguments = [\n x.name\n for x in test_macro.nodes # type: ignore[attr-defined]\n if isinstance(x, jinja2.nodes.Name)\n ]\n\n # macro_arguments: List of args parsed from macro SQL\n # macro.arguments: List of args manually added to the properties file\n\n non_complying_args = []\n for arg in macro_arguments:\n macro_doc_raw = [x for x in macro.arguments if x.name == arg]\n if macro_doc_raw == [] or (\n arg not in [x.name for x in macro.arguments]\n or len(macro_doc_raw[0].description.strip()) <= 4\n ):\n non_complying_args.append(arg)\n\n assert (\n non_complying_args == []\n ), f\"Macro `{macro.name}` does not have a populated description for the following argument(s): {non_complying_args}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_code_does_not_contain_regexp_pattern(\n macro: \"Macros\",\n regexp_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"The raw code for a macro must not match the specified regexp pattern.\n\n Parameters:\n macro (Macros): The Macros object to check.\n regexp_pattern (str): The regexp pattern that should not be matched by the macro code.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Prefer `coalesce` over `ifnull`: https://docs.sqlfluff.com/en/stable/rules.html#sqlfluff.rules.sphinx.Rule_CV02\n - name: check_macro_code_does_not_contain_regexp_pattern\n regexp_pattern: .*[i][f][n][u][l][l].*\n ```\n\n \"\"\"\n assert (\n re.compile(regexp_pattern.strip(), flags=re.DOTALL).match(macro.macro_sql)\n is None\n ), f\"Macro `{macro.name}` contains a banned string: `{regexp_pattern.strip()}`.\"\n
# Only \"common\" macros need to have a populated description\nmanifest_checks:\n - name: check_macro_description_populated\n include: ^macros/common\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_description_populated(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macros must have a populated description.\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_description_populated\n ```\n ```yaml\n # Only \"common\" macros need to have a populated description\n manifest_checks:\n - name: check_macro_description_populated\n include: ^macros/common\n ```\n\n \"\"\"\n assert (\n len(macro.description.strip()) > 4\n ), f\"Macro `{macro.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_max_number_of_lines(\n macro: \"Macros\",\n max_number_of_lines: int = 50,\n **kwargs,\n) -> None:\n \"\"\"Macros may not have more than the specified number of lines.\n\n Parameters:\n macro (Macros): The Macros object to check.\n max_number_of_lines (int): The maximum number of permitted lines.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_max_number_of_lines\n ```\n ```yaml\n manifest_checks:\n - name: check_macro_max_number_of_lines\n max_number_of_lines: 100\n ```\n\n \"\"\"\n actual_number_of_lines = macro.macro_sql.count(\"\\n\") + 1\n\n assert (\n actual_number_of_lines <= max_number_of_lines\n ), f\"Macro `{macro.name}` has {actual_number_of_lines} lines, this is more than the maximum permitted number of lines ({max_number_of_lines}).\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_name_matches_file_name(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macros names must be the same as the file they are contained in.\n\n Generic tests are also macros, however to document these tests the \"name\" value must be preceded with \"test_\".\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_name_matches_file_name\n ```\n\n \"\"\"\n if macro.name.startswith(\"test_\"):\n assert (\n macro.name[5:] == macro.original_file_path.split(\"/\")[-1].split(\".\")[0]\n ), f\"Macro `{macro.unique_id}` is not in a file named `{macro.name[5:]}.sql`.\"\n else:\n assert (\n macro.name == macro.original_file_path.split(\"/\")[-1].split(\".\")[0]\n ), f\"Macro `{macro.name}` is not in a file of the same name.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_macros.py
def check_macro_property_file_location(macro: \"Macros\", **kwargs) -> None:\n \"\"\"Macro properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/5-the-rest-of-the-project#how-we-use-the-other-folders).\n\n Parameters:\n macro (Macros): The Macros object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the macro path. Macro paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the macro path. Only macro paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_macro_property_file_location\n ```\n\n \"\"\"\n expected_substr = \"_\".join(macro.original_file_path[6:].split(\"/\")[:-1])\n properties_yml_name = macro.patch_path.split(\"/\")[-1]\n\n if macro.original_file_path.startswith(\n \"tests/\",\n ): # Do not check generic tests (which are also macros)\n pass\n elif expected_substr == \"\": # i.e. macro in ./macros\n assert (\n properties_yml_name == \"_macros.yml\"\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) should be `_macros.yml`.\"\n else:\n assert properties_yml_name.startswith(\n \"_\",\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) does not start with an underscore.\"\n assert (\n expected_substr in properties_yml_name\n ), f\"The properties file for `{macro.name}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substr}`).\"\n assert properties_yml_name.endswith(\n \"__macros.yml\",\n ), f\"The properties file for `{macro.name.name}` (`{properties_yml_name}`) does not end with `__macros.yml`.\"\n
Enforce that the name of the dbt project matches a supplied regex. Generally used to enforce that project names conform to something like company_<DOMAIN>.
Parameters:
Name Type Description Default manifest_objDbtBouncerManifest
The manifest object.
required project_name_patternstr
Regex pattern to match the project name.
required
Other Parameters (passed via config file):
Name Type Description severityOptional[Literal['error', 'warn']]
Source code in src/dbt_bouncer/checks/manifest/check_metadata.py
def check_project_name(\n manifest_obj: \"DbtBouncerManifest\",\n project_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Enforce that the name of the dbt project matches a supplied regex. Generally used to enforce that project names conform to something like `company_<DOMAIN>`.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The manifest object.\n project_name_pattern (str): Regex pattern to match the project name.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_project_name\n project_name_pattern: ^awesome_company_\n ```\n\n \"\"\"\n assert (\n re.compile(project_name_pattern.strip()).match(\n manifest_obj.manifest.metadata.project_name,\n )\n is not None\n ), f\"Project name (`{manifest_obj.manifest.metadata.project_name}`) does not conform to the supplied regex `({project_name_pattern.strip()})`.\"\n
Models must have the specified access attribute. Requires dbt 1.7+.
Parameters:
Name Type Description Default accessLiteral['private', 'protected', 'public']
The access level to check for.
required modelDbtBouncerModel
The DbtBouncerModel object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the model path. Model paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the model path. Only model paths that match the pattern will be checked.
severityOptional[Literal['error', 'warn']]
Severity level of the check. Default: error.
Example(s):
manifest_checks:\n # Align with dbt best practices that marts should be `public`, everything else should be `protected`\n - name: check_model_access\n access: protected\n include: ^models/intermediate\n - name: check_model_access\n access: public\n include: ^models/marts\n - name: check_model_access\n access: protected\n include: ^models/staging\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_access(\n access: str,\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"Models must have the specified access attribute. Requires dbt 1.7+.\n\n Parameters:\n access (Literal[\"private\", \"protected\", \"public\"]): The access level to check for.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Align with dbt best practices that marts should be `public`, everything else should be `protected`\n - name: check_model_access\n access: protected\n include: ^models/intermediate\n - name: check_model_access\n access: public\n include: ^models/marts\n - name: check_model_access\n access: protected\n include: ^models/staging\n ```\n\n \"\"\"\n assert (\n model.access.value == access\n ), f\"`{model.name}` has `{model.access.value}` access, it should have access `{access}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_contract_enforced_for_public_model(\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"Public models must have contracts enforced.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_contract_enforced_for_public_model\n ```\n\n \"\"\"\n if model.access.value == \"public\":\n assert (\n model.contract.enforced is True\n ), f\"`{model.name}` is a public model but does not have contracts enforced.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_description_populated(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Models must have a populated description.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_description_populated\n ```\n\n \"\"\"\n assert (\n len(model.description.strip()) > 4\n ), f\"`{model.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_documentation_coverage(\n models: List[\"DbtBouncerModel\"],\n min_model_documentation_coverage_pct: float = 100,\n **kwargs,\n) -> None:\n \"\"\"Set the minimum percentage of models that have a populated description.\n\n Parameters:\n min_model_documentation_coverage_pct (float): The minimum percentage of models that must have a populated description.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_description_populated\n min_model_documentation_coverage_pct: 90\n ```\n\n \"\"\"\n num_models = len(models)\n models_with_description = []\n for model in models:\n if len(model.description.strip()) > 4:\n models_with_description.append(model.unique_id)\n\n num_models_with_descriptions = len(models_with_description)\n model_description_coverage_pct = (num_models_with_descriptions / num_models) * 100\n\n assert (\n model_description_coverage_pct >= min_model_documentation_coverage_pct\n ), f\"Only {model_description_coverage_pct}% of models have a populated description, this is less than the permitted minimum of {min_model_documentation_coverage_pct}%.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_documented_in_same_directory(\n model: \"DbtBouncerModel\", **kwargs\n) -> None:\n \"\"\"Models must be documented in the same directory where they are defined (i.e. `.yml` and `.sql` files are in the same directory).\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_documented_in_same_directory\n ```\n\n \"\"\"\n model_doc_dir = model.patch_path[model.patch_path.find(\"models\") :].split(\"/\")[:-1]\n model_sql_dir = model.original_file_path.split(\"/\")[:-1]\n\n assert (\n model_doc_dir == model_sql_dir\n ), f\"`{model.name}` is documented in a different directory to the `.sql` file: `{'/'.join(model_doc_dir)}` vs `{'/'.join(model_sql_dir)}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_code_does_not_contain_regexp_pattern(\n model: \"DbtBouncerModel\",\n regexp_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"The raw code for a model must not match the specified regexp pattern.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n regexp_pattern (str): The regexp pattern that should not be matched by the model code.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n # Prefer `coalesce` over `ifnull`: https://docs.sqlfluff.com/en/stable/rules.html#sqlfluff.rules.sphinx.Rule_CV02\n - name: check_model_code_does_not_contain_regexp_pattern\n regexp_pattern: .*[i][f][n][u][l][l].*\n ```\n\n \"\"\"\n assert (\n re.compile(regexp_pattern.strip(), flags=re.DOTALL).match(model.raw_code)\n is None\n ), f\"`{model.name}` contains a banned string: `{regexp_pattern.strip()}`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_depends_on_multiple_sources(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Models cannot reference more than one source.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_depends_on_multiple_sources\n ```\n\n \"\"\"\n num_reffed_sources = sum(\n x.split(\".\")[0] == \"source\" for x in model.depends_on.nodes\n )\n assert num_reffed_sources <= 1, f\"`{model.name}` references more than one source.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_contracts_enforced(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Model must have contracts enforced.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_contracts_enforced\n include: ^models/marts\n ```\n\n \"\"\"\n assert (\n model.contract.enforced is True\n ), f\"`{model.name}` does not have contracts enforced.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_meta_keys(\n keys: NestedDict,\n model: \"DbtBouncerModel\",\n **kwargs,\n) -> None:\n \"\"\"The `meta` config for models must have the specified keys.\n\n Parameters:\n keys (NestedDict): A list (that may contain sub-lists) of required keys.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_meta_keys\n keys:\n - maturity\n - owner\n ```\n\n \"\"\"\n missing_keys = find_missing_meta_keys(\n meta_config=model.meta,\n required_keys=keys,\n )\n assert (\n missing_keys == []\n ), f\"`{model.name}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_no_upstream_dependencies(\n model: \"DbtBouncerModel\", **kwargs\n) -> None:\n \"\"\"Identify if models have no upstream dependencies as this likely indicates hard-coded tables references.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_no_upstream_dependencies\n ```\n\n \"\"\"\n assert (\n len(model.depends_on.nodes) > 0\n ), f\"`{model.name}` has no upstream dependencies, this likely indicates hard-coded tables references.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_tags(model: \"DbtBouncerModel\", tags: List[str], **kwargs) -> None:\n \"\"\"Models must have the specified tags.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n tags (List[str]): List of tags to check for.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_tags\n tags:\n - tag_1\n - tag_2\n ```\n\n \"\"\"\n missing_tags = [tag for tag in tags if tag not in model.tags]\n assert not missing_tags, f\"`{model.name}` is missing required tags: {missing_tags}.\"\n
manifest_checks:\n # Example of allowing a custom uniqueness test\n - name: check_model_has_unique_test\n accepted_uniqueness_tests:\n - expect_compound_columns_to_be_unique\n - my_custom_uniqueness_test\n - unique\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_unique_test(\n model: \"DbtBouncerModel\",\n tests: \"DbtBouncerTest\",\n accepted_uniqueness_tests: List[str] = (\n [ # noqa: B006\n \"expect_compound_columns_to_be_unique\",\n \"dbt_utils.unique_combination_of_columns\",\n \"unique\",\n ]\n ),\n **kwargs,\n) -> None:\n \"\"\"Models must have a test for uniqueness of a column.\n\n Parameters:\n accepted_uniqueness_tests (Optional[List[str]]): List of tests that are accepted as uniqueness tests.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_unique_test\n include: ^models/marts\n ```\n ```yaml\n manifest_checks:\n # Example of allowing a custom uniqueness test\n - name: check_model_has_unique_test\n accepted_uniqueness_tests:\n - expect_compound_columns_to_be_unique\n - my_custom_uniqueness_test\n - unique\n ```\n\n \"\"\"\n num_unique_tests = sum(\n test.attached_node == model.unique_id\n and test.test_metadata.name in accepted_uniqueness_tests\n for test in tests\n )\n assert (\n num_unique_tests >= 1\n ), f\"`{model.name}` does not have a test for uniqueness of a column.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_has_unit_tests(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n unit_tests: List[\"UnitTests\"],\n min_number_of_unit_tests: int = 1,\n **kwargs,\n) -> None:\n \"\"\"Models must have more than the specified number of unit tests.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n min_number_of_unit_tests (Optional[int]): The minimum number of unit tests that a model must have.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n unit_tests (List[UnitTests]): List of UnitTests objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_has_unit_tests\n include: ^models/marts\n ```\n ```yaml\n manifest_checks:\n - name: check_model_has_unit_tests\n min_number_of_unit_tests: 2\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n num_unit_tests = len(\n [\n t.unique_id\n for t in unit_tests\n if t.depends_on.nodes[0] == model.unique_id\n ],\n )\n assert (\n num_unit_tests >= min_number_of_unit_tests\n ), f\"`{model.name}` has {num_unit_tests} unit tests, this is less than the minimum of {min_number_of_unit_tests}.\"\n else:\n logging.warning(\n \"The `check_model_has_unit_tests` check is only supported for dbt 1.8.0 and above.\",\n )\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_chained_views(\n manifest_obj: \"DbtBouncerManifest\",\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n materializations_to_include: List[str] = [\"ephemeral\", \"view\"], # noqa: B006\n max_chained_views: int = 3,\n **kwargs,\n) -> None:\n \"\"\"Models cannot have more than the specified number of upstream dependents that are not tables.\n\n Parameters:\n materializations_to_include (Optional[List[str]]): List of materializations to include in the check.\n max_chained_views (Optional[int]): The maximum number of upstream dependents that are not tables.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_chained_views\n ```\n ```yaml\n manifest_checks:\n - name: check_model_max_chained_views\n materializations_to_include:\n - ephemeral\n - my_custom_materialization\n - view\n max_chained_views: 5\n ```\n\n \"\"\"\n\n def return_upstream_view_models(\n materializations,\n max_chained_views,\n models,\n model_unique_ids_to_check,\n package_name,\n depth=0,\n ):\n \"\"\"Recursive function to return model unique_id's of upstream models that are views. Depth of recursion can be specified. If no models meet the criteria then an empty list is returned.\n\n Returns\n -\n List[str]: List of model unique_id's of upstream models that are views.\n\n \"\"\"\n if depth == max_chained_views or model_unique_ids_to_check == []:\n return model_unique_ids_to_check\n\n relevant_upstream_models = []\n for model in model_unique_ids_to_check:\n upstream_nodes = list(\n next(m2 for m2 in models if m2.unique_id == model).depends_on.nodes,\n )\n if upstream_nodes != []:\n upstream_models = [\n m\n for m in upstream_nodes\n if m.split(\".\")[0] == \"model\" and m.split(\".\")[1] == package_name\n ]\n for i in upstream_models:\n if (\n next(m for m in models if m.unique_id == i).config.materialized\n in materializations\n ):\n relevant_upstream_models.append(i)\n\n depth += 1\n return return_upstream_view_models(\n materializations=materializations,\n max_chained_views=max_chained_views,\n models=models,\n model_unique_ids_to_check=relevant_upstream_models,\n package_name=package_name,\n depth=depth,\n )\n\n assert (\n len(\n return_upstream_view_models(\n materializations=materializations_to_include,\n max_chained_views=max_chained_views,\n models=models,\n model_unique_ids_to_check=[model.unique_id],\n package_name=manifest_obj.manifest.metadata.project_name,\n ),\n )\n == 0\n ), f\"`{model.name}` has more than {max_chained_views} upstream dependents that are not tables.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_fanout(\n model: \"DbtBouncerModel\",\n models: List[\"DbtBouncerModel\"],\n max_downstream_models: int = 3,\n **kwargs,\n) -> None:\n \"\"\"Models cannot have more than the specified number of downstream models.\n\n Parameters:\n max_downstream_models (Optional[int]): The maximum number of permitted downstream models.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_fanout\n max_downstream_models: 2\n ```\n\n \"\"\"\n num_downstream_models = sum(model.unique_id in m.depends_on.nodes for m in models)\n\n assert (\n num_downstream_models <= max_downstream_models\n ), f\"`{model.name}` has {num_downstream_models} downstream models, which is more than the permitted maximum of {max_downstream_models}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_number_of_lines(\n model: \"DbtBouncerModel\",\n max_number_of_lines: int = 100,\n **kwargs,\n) -> None:\n \"\"\"Models may not have more than the specified number of lines.\n\n Parameters:\n max_number_of_lines (int): The maximum number of permitted lines.\n\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_number_of_lines\n ```\n ```yaml\n manifest_checks:\n - name: check_model_max_number_of_lines\n max_number_of_lines: 150\n ```\n\n \"\"\"\n actual_number_of_lines = model.raw_code.count(\"\\n\") + 1\n\n assert (\n actual_number_of_lines <= max_number_of_lines\n ), f\"`{model.name}` has {actual_number_of_lines} lines, this is more than the maximum permitted number of lines ({max_number_of_lines}).\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_max_upstream_dependencies(\n model: \"DbtBouncerModel\",\n max_upstream_macros: int = 5,\n max_upstream_models: int = 5,\n max_upstream_sources: int = 1,\n **kwargs,\n) -> None:\n \"\"\"Limit the number of upstream dependencies a model has.\n\n Parameters:\n max_upstream_macros (Optional[int]): The maximum number of permitted upstream macros.\n max_upstream_models (Optional[int]): The maximum number of permitted upstream models.\n max_upstream_sources (Optional[int]): The maximum number of permitted upstream sources.\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_max_upstream_dependencies\n max_upstream_models: 3\n ```\n\n \"\"\"\n num_upstream_macros = len(list(model.depends_on.macros))\n num_upstream_models = len(\n [m for m in model.depends_on.nodes if m.split(\".\")[0] == \"model\"],\n )\n num_upstream_sources = len(\n [m for m in model.depends_on.nodes if m.split(\".\")[0] == \"source\"],\n )\n\n assert (\n num_upstream_macros <= max_upstream_macros\n ), f\"`{model.name}` has {num_upstream_macros} upstream macros, which is more than the permitted maximum of {max_upstream_macros}.\"\n assert (\n num_upstream_models <= max_upstream_models\n ), f\"`{model.name}` has {num_upstream_models} upstream models, which is more than the permitted maximum of {max_upstream_models}.\"\n assert (\n num_upstream_sources <= max_upstream_sources\n ), f\"`{model.name}` has {num_upstream_sources} upstream sources, which is more than the permitted maximum of {max_upstream_sources}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_names(\n model: \"DbtBouncerModel\",\n model_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Models must have a name that matches the supplied regex.\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n model_name_pattern (str): Regexp the model name must match.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_names\n include: ^models/intermediate\n model_name_pattern: ^int_\n - name: check_model_names\n include: ^models/staging\n model_name_pattern: ^stg_\n ```\n\n \"\"\"\n assert (\n re.compile(model_name_pattern.strip()).match(model.name) is not None\n ), f\"`{model.name}` does not match the supplied regex `{model_name_pattern.strip()})`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_property_file_location(model: \"DbtBouncerModel\", **kwargs) -> None:\n \"\"\"Model properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/1-guide-overview).\n\n Parameters:\n model (DbtBouncerModel): The DbtBouncerModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the model path. Model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the model path. Only model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_property_file_location\n ```\n\n \"\"\"\n expected_substr = (\n \"_\".join(model.original_file_path.split(\"/\")[1:-1])\n .replace(\"staging\", \"stg\")\n .replace(\"intermediate\", \"int\")\n .replace(\"marts\", \"\")\n )\n properties_yml_name = model.patch_path.split(\"/\")[-1]\n\n assert properties_yml_name.startswith(\n \"_\",\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not start with an underscore.\"\n assert (\n expected_substr in properties_yml_name\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substr}`).\"\n assert properties_yml_name.endswith(\n \"__models.yml\",\n ), f\"The properties file for `{model.name}` (`{properties_yml_name}`) does not end with `__models.yml`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_models.py
def check_model_test_coverage(\n models: List[\"DbtBouncerModel\"],\n tests: List[\"DbtBouncerTest\"],\n min_model_test_coverage_pct: float = 100,\n **kwargs,\n) -> None:\n \"\"\"Set the minimum percentage of models that have at least one test.\n\n Parameters:\n min_model_test_coverage_pct (float): The minimum percentage of models that must have at least one test.\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n tests (List[DbtBouncerTest]): List of DbtBouncerTest objects parsed from `manifest.json`.\n\n Other Parameters:\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_model_test_coverage\n min_model_test_coverage_pct: 90\n ```\n\n \"\"\"\n num_models = len(models)\n models_with_tests = []\n for model in models:\n for test in tests:\n if model.unique_id in test.depends_on.nodes:\n models_with_tests.append(model.unique_id)\n num_models_with_tests = len(set(models_with_tests))\n model_test_coverage_pct = (num_models_with_tests / num_models) * 100\n\n assert (\n model_test_coverage_pct >= min_model_test_coverage_pct\n ), f\"Only {model_test_coverage_pct}% of models have at least one test, this is less than the permitted minimum of {min_model_test_coverage_pct}%.\"\n
Semantic models should be based on public models only.
Parameters:
Name Type Description Default modelsList[DbtBouncerModel]
List of DbtBouncerModel objects parsed from manifest.json.
required semantic_modelDbtBouncerSemanticModel
The DbtBouncerSemanticModel object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Semantic model paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Only semantic model paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_semantic_models.py
def check_semantic_model_based_on_non_public_models(\n models: List[\"DbtBouncerModel\"],\n semantic_model: \"DbtBouncerSemanticModel\",\n **kwargs,\n) -> None:\n \"\"\"Semantic models should be based on public models only.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n semantic_model (DbtBouncerSemanticModel): The DbtBouncerSemanticModel object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Semantic model paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the semantic model path (i.e the .yml file where the semantic model is configured). Only semantic model paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_semantic_model_based_on_non_public_models\n ```\n\n \"\"\"\n non_public_upstream_dependencies = []\n for model in semantic_model.depends_on.nodes:\n if (\n model.split(\".\")[0] == \"model\"\n and model.split(\".\")[1] == semantic_model.package_name\n ):\n model = next(m for m in models if m.unique_id == model)\n if model.access.value != \"public\":\n non_public_upstream_dependencies.append(model.name)\n\n assert not non_public_upstream_dependencies, f\"Semantic model `{semantic_model.name}` is based on a model(s) that is not public: {non_public_upstream_dependencies}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_description_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated description.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_description_populated\n ```\n\n \"\"\"\n assert (\n len(source.description.strip()) > 4\n ), f\"`{source.source_name}.{source.name}` does not have a populated description.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_freshness_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated freshness.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_freshness_populated\n ```\n\n \"\"\"\n assert (\n source.freshness.error_after.count is not None\n and source.freshness.error_after.period is not None\n ) or (\n source.freshness.warn_after.count is not None\n and source.freshness.warn_after.period is not None\n ), f\"`{source.source_name}.{source.name}` does not have a populated freshness.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_has_meta_keys(\n keys: \"NestedDict\",\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"The `meta` config for sources must have the specified keys.\n\n Parameters:\n keys (NestedDict): A list (that may contain sub-lists) of required keys.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_has_meta_keys\n keys:\n - contact:\n - email\n - slack\n - owner\n ```\n\n \"\"\"\n missing_keys = find_missing_meta_keys(\n meta_config=source.meta,\n required_keys=keys,\n )\n assert (\n missing_keys == []\n ), f\"`{source.source_name}.{source.name}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_has_tags(\n source: \"DbtBouncerSource\",\n tags: List[str],\n **kwargs,\n) -> None:\n \"\"\"Sources must have the specified tags.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n tags (List[str]): List of tags to check for.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_has_tags\n tags:\n - tag_1\n - tag_2\n ```\n\n \"\"\"\n missing_tags = [tag for tag in tags if tag not in source.tags]\n assert (\n not missing_tags\n ), f\"`{source.source_name}.{source.name}` is missing required tags: {missing_tags}.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_loader_populated(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Sources must have a populated loader.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_loader_populated\n ```\n\n \"\"\"\n assert (\n source.loader != \"\"\n ), f\"`{source.source_name}.{source.name}` does not have a populated loader.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_names(\n source: \"DbtBouncerSource\",\n source_name_pattern: str,\n **kwargs,\n) -> None:\n \"\"\"Sources must have a name that matches the supplied regex.\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n source_name_pattern (str): Regexp the source name must match.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_names\n source_name_pattern: >\n ^[a-z0-9_]*$\n ```\n\n \"\"\"\n assert (\n re.compile(source_name_pattern.strip()).match(source.name) is not None\n ), f\"`{source.source_name}.{source.name}` does not match the supplied regex `({source_name_pattern.strip()})`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_not_orphaned(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Sources must be referenced in at least one model.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_not_orphaned\n ```\n\n \"\"\"\n num_refs = sum(source.unique_id in model.depends_on.nodes for model in models)\n assert (\n num_refs >= 1\n ), f\"Source `{source.source_name}.{source.name}` is orphaned, i.e. not referenced by any model.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_property_file_location(source: \"DbtBouncerSource\", **kwargs) -> None:\n \"\"\"Source properties files must follow the guidance provided by dbt [here](https://docs.getdbt.com/best-practices/how-we-structure/1-guide-overview).\n\n Parameters:\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_property_file_location\n ```\n\n \"\"\"\n path_cleaned = source.original_file_path.replace(\"models/staging\", \"\")\n expected_substring = \"_\".join(path_cleaned.split(\"/\")[:-1])\n\n assert path_cleaned.split(\n \"/\",\n )[\n -1\n ].startswith(\n \"_\",\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not start with an underscore.\"\n assert (\n expected_substring in path_cleaned\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not contain the expected substring (`{expected_substring}`).\"\n assert path_cleaned.split(\n \"/\",\n )[\n -1\n ].endswith(\n \"__sources.yml\",\n ), f\"The properties file for `{source.source_name}.{source.name}` (`{path_cleaned}`) does not end with `__sources.yml`.\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_used_by_models_in_same_directory(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Sources can only be referenced by models that are located in the same directory where the source is defined.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_used_by_models_in_same_directory\n ```\n\n \"\"\"\n reffed_models_not_in_same_dir = []\n for model in models:\n if (\n source.unique_id in model.depends_on.nodes\n and model.original_file_path.split(\"/\")[:-1]\n != source.original_file_path.split(\"/\")[:-1]\n ):\n reffed_models_not_in_same_dir.append(model.unique_id.split(\".\")[0])\n\n assert (\n len(reffed_models_not_in_same_dir) == 0\n ), f\"Source `{source.source_name}.{source.name}` is referenced by models defined in a different directory: {reffed_models_not_in_same_dir}\"\n
Source code in src/dbt_bouncer/checks/manifest/check_sources.py
def check_source_used_by_only_one_model(\n models: List[\"DbtBouncerModel\"],\n source: \"DbtBouncerSource\",\n **kwargs,\n) -> None:\n \"\"\"Each source can be referenced by a maximum of one model.\n\n Parameters:\n models (List[DbtBouncerModel]): List of DbtBouncerModel objects parsed from `manifest.json`.\n source (DbtBouncerSource): The DbtBouncerSource object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_source_used_by_only_one_model\n ```\n\n \"\"\"\n num_refs = sum(source.unique_id in model.depends_on.nodes for model in models)\n assert (\n num_refs <= 1\n ), f\"Source `{source.source_name}.{source.name}` is referenced by more than one model.\"\n
"},{"location":"checks/manifest/check_unit_tests/","title":"Manifest Checks: Unit Tests","text":"
Note
The below checks require manifest.json to be present.
A list of formats that are allowed to be used for expect input in a unit test.
['csv', 'dict', 'sql']unit_testUnitTests
The UnitTests object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_unit_tests.py
def check_unit_test_expect_format(\n manifest_obj: \"DbtBouncerManifest\",\n unit_test: \"UnitTests\",\n permitted_formats: List[Literal[\"csv\", \"dict\", \"sql\"]] = [\"csv\", \"dict\", \"sql\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Unit tests can only use the specified formats.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n permitted_formats (Optional[List[Literal[\"csv\", \"dict\", \"sql\"]]]): A list of formats that are allowed to be used for `expect` input in a unit test.\n unit_test (UnitTests): The UnitTests object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_unit_test_expect_format\n permitted_formats:\n - csv\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n assert (\n unit_test.expect.format.value in permitted_formats\n ), f\"Unit test `{unit_test.name}` has an `expect` format that is not permitted. Permitted formats are: {permitted_formats}.\"\n else:\n logging.warning(\n \"The `check_unit_test_expect_format` check is only supported for dbt 1.8.0 and above.\",\n )\n
A list of formats that are allowed to be used for expect input in a unit test.
['csv', 'dict', 'sql']unit_testUnitTests
The UnitTests object to check.
required
Other Parameters (passed via config file):
Name Type Description excludeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
includeOptional[str]
Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
Source code in src/dbt_bouncer/checks/manifest/check_unit_tests.py
def check_unit_test_given_formats(\n manifest_obj: \"DbtBouncerManifest\",\n unit_test: \"UnitTests\",\n permitted_formats: List[Literal[\"csv\", \"dict\", \"sql\"]] = [\"csv\", \"dict\", \"sql\"], # noqa: B006\n **kwargs,\n) -> None:\n \"\"\"Unit tests can only use the specified formats.\n\n !!! warning\n\n This check is only supported for dbt 1.8.0 and above.\n\n Parameters:\n manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.\n permitted_formats (Optional[List[Literal[\"csv\", \"dict\", \"sql\"]]]): A list of formats that are allowed to be used for `expect` input in a unit test.\n unit_test (UnitTests): The UnitTests object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n manifest_checks:\n - name: check_unit_test_given_formats\n permitted_formats:\n - csv\n ```\n\n \"\"\"\n if semver.Version.parse(manifest_obj.manifest.metadata.dbt_version) >= \"1.8.0\":\n given_formats = [i.format.value for i in unit_test.given]\n assert all(\n e in permitted_formats for e in given_formats\n ), f\"Unit test `{unit_test.name}` has given formats which are not permitted. Permitted formats are: {permitted_formats}.\"\n else:\n logging.warning(\n \"The `check_unit_test_given_formats` check is only supported for dbt 1.8.0 and above.\",\n )\n
Source code in src/dbt_bouncer/checks/run_results/check_run_results.py
def check_run_results_max_gigabytes_billed(\n max_gigabytes_billed: float,\n run_result: \"DbtBouncerRunResult\",\n **kwargs,\n) -> None:\n \"\"\"Each result can have a maximum number of gigabytes billed.\n\n !!! note\n\n Note that this check only works for the `dbt-bigquery` adapter.\n\n Parameters:\n max_gigabytes_billed (float): The maximum number of gigabytes billed.\n run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Raises: # noqa:DOC502\n KeyError: If the `dbt-bigquery` adapter is not used.\n\n Example(s):\n ```yaml\n run_results_checks:\n - name: check_run_results_max_gigabytes_billed\n max_gigabytes_billed: 100\n ```\n\n \"\"\"\n try:\n gigabytes_billed = run_result.adapter_response[\"bytes_billed\"] / (1000**3)\n except KeyError as e:\n raise RuntimeError( # noqa: DOC501\n \"`bytes_billed` not found in adapter response. Are you using the `dbt-bigquery` adapter?\",\n ) from e\n\n assert (\n gigabytes_billed < max_gigabytes_billed\n ), f\"`{run_result.unique_id.split('.')[-2]}` results in ({gigabytes_billed} billed bytes, this is greater than permitted ({max_gigabytes_billed}).\"\n
run_results_checks:\n - name: check_run_results_max_execution_time\n include: ^models/staging # Not a good idea, here for demonstration purposes only\n max_execution_time_seconds: 10\n
Source code in src/dbt_bouncer/checks/run_results/check_run_results.py
def check_run_results_max_execution_time(\n max_execution_time_seconds: float,\n run_result: \"DbtBouncerRunResult\",\n **kwargs,\n) -> None:\n \"\"\"Each result can take a maximum duration (seconds).\n\n Parameters:\n max_execution_time_seconds (float): The maximum execution time (seconds) allowed for a node.\n run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.\n\n Other Parameters:\n exclude (Optional[str]): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.\n include (Optional[str]): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.\n severity (Optional[Literal[\"error\", \"warn\"]]): Severity level of the check. Default: `error`.\n\n Example(s):\n ```yaml\n run_results_checks:\n - name: check_run_results_max_execution_time\n max_execution_time_seconds: 60\n ```\n ```yaml\n run_results_checks:\n - name: check_run_results_max_execution_time\n include: ^models/staging # Not a good idea, here for demonstration purposes only\n max_execution_time_seconds: 10\n ```\n\n \"\"\"\n assert (\n run_result.execution_time <= max_execution_time_seconds\n ), f\"`{run_result.unique_id.split('.')[-1]}` has an execution time ({run_result.execution_time} greater than permitted ({max_execution_time_seconds}s).\"\n
"}]}
\ No newline at end of file
diff --git a/latest/sitemap.xml b/latest/sitemap.xml
index 24b38f1a..c520d95b 100644
--- a/latest/sitemap.xml
+++ b/latest/sitemap.xml
@@ -70,6 +70,11 @@
2024-08-30daily
+
+ https://godatadriven.github.io/dbt-bouncer/latest/checks/manifest/check_semantic_models/
+ 2024-08-30
+ daily
+ https://godatadriven.github.io/dbt-bouncer/latest/checks/manifest/check_sources/2024-08-30
diff --git a/latest/sitemap.xml.gz b/latest/sitemap.xml.gz
index 0fc071070aee0bd0de7112b5e4c12d5719609fb1..13e3d1fd16f794c39d2ef08ea8c6daabea6d426a 100644
GIT binary patch
delta 370
zcmV-&0ge9n0)YdN7=N*D!ypib_dJE=-NsEjq?O}Ht<)+@DpkpB5jbO13<(@b`u4?f
zR9T(d0kfDpAHV)jX8|r(mrf-ouudAiOz-j{O+Yilr54L{w|&hP>8hHQ-YN%}gktTM
zX^VJxBIJBN=T0JY^x$%11#zqmfg`shDe}j(nk8ZM0>kV&N`GZo)vLH5BMhfVxs@l-
zxscfUeJ%~*d(8I6YX+7mir|oxq+d6dp$N!>&;tCzL}Cy*)_V6qHd&um~1!n7?WKAh+#vh9a4+H
zqxwKvV4p5R$7?K-WN%jPGp;c_s}TbKZfk{YOh_(3|#
zteHSVItZ(S!URYR2Q`6?xNxo2DA{y?qCHQ3fp{&k?%&7AV!f`z5#m*FKj4fr|ITxL
Qgx|aB8$Hmjr49}N06%@Q+5i9m
delta 361
zcmV-v0ha!O1NZ`v7=J-)!ypvL@BI`Z_r$E*p&Mxm8*KE_!D!FH>Wh&XZS~o9-+rbk
zBka5#j2C^+=bzsn<$>ku(y8PG)=8t6>0Mr=320`x)MAkup4@e30DAOXb81KY7ux;
z52OY5=^`X%p<-MaF@*Iks-5wc!36vm1e_QSY63SBZ~Oga#A}IlzaOI)>vbJEu2+G6urtp5yPxY6enZtaGwL6s
HJq-W=Q}wUY