From 95556b984b300ee7ecb0a44d3f2ddfdd8a26e04b Mon Sep 17 00:00:00 2001 From: FouziaTariq <86288319+FouziaTariq@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:15:38 +0100 Subject: [PATCH 01/10] Make PostgresUserPasswordProfileMapping schema profile argument schema optional. Issue #675 --- cosmos/profiles/postgres/user_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 731d60079..804751afb 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -21,7 +21,7 @@ class PostgresUserPasswordProfileMapping(BaseProfileMapping): "user", "password", "dbname", - "schema", + ] secret_fields = [ "password", From 69f2597c301398f4d32b1f470faa84ef76ab7984 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Nov 2023 20:19:03 +0000 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20f?= =?UTF-8?q?ormat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cosmos/profiles/postgres/user_pass.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 804751afb..22e47bc6a 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -21,7 +21,6 @@ class PostgresUserPasswordProfileMapping(BaseProfileMapping): "user", "password", "dbname", - ] secret_fields = [ "password", From 8c934c0a88688c55ad2316ff3622a39fcf63e6fa Mon Sep 17 00:00:00 2001 From: FouziaTariq <86288319+FouziaTariq@users.noreply.github.com> Date: Fri, 17 Nov 2023 21:47:12 +0100 Subject: [PATCH 03/10] Update test_pg_user_pass.py --- tests/profiles/postgres/test_pg_user_pass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/profiles/postgres/test_pg_user_pass.py b/tests/profiles/postgres/test_pg_user_pass.py index 3492450b5..ca6af2506 100644 --- a/tests/profiles/postgres/test_pg_user_pass.py +++ b/tests/profiles/postgres/test_pg_user_pass.py @@ -83,9 +83,9 @@ def test_connection_claiming() -> None: assert not profile_mapping.can_claim_connection() # also test when there's no schema - conn = Connection(**potential_values) # type: ignore + conn = Connection(**{k: v for k, v in potential_values.items() if k != 'schema'}) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): - profile_mapping = PostgresUserPasswordProfileMapping(conn, {}) + profile_mapping = PostgresUserPasswordProfileMapping(conn, {"schema": None}) assert not profile_mapping.can_claim_connection() # if we have them all, it should claim From 07b2e61920e5df959f19b28149f0acc85baafe4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Nov 2023 20:47:25 +0000 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20f?= =?UTF-8?q?ormat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/profiles/postgres/test_pg_user_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/profiles/postgres/test_pg_user_pass.py b/tests/profiles/postgres/test_pg_user_pass.py index ca6af2506..174fff139 100644 --- a/tests/profiles/postgres/test_pg_user_pass.py +++ b/tests/profiles/postgres/test_pg_user_pass.py @@ -83,7 +83,7 @@ def test_connection_claiming() -> None: assert not profile_mapping.can_claim_connection() # also test when there's no schema - conn = Connection(**{k: v for k, v in potential_values.items() if k != 'schema'}) + conn = Connection(**{k: v for k, v in potential_values.items() if k != "schema"}) with patch("airflow.hooks.base.BaseHook.get_connection", return_value=conn): profile_mapping = PostgresUserPasswordProfileMapping(conn, {"schema": None}) assert not profile_mapping.can_claim_connection() From e496cca169287b3f6255c90600268dff2840ba98 Mon Sep 17 00:00:00 2001 From: FouziaTariq <86288319+FouziaTariq@users.noreply.github.com> Date: Sat, 25 Nov 2023 22:16:01 +0100 Subject: [PATCH 05/10] Updated user_pass.py --- cosmos/profiles/postgres/user_pass.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 22e47bc6a..b0c785d20 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -46,6 +46,9 @@ def profile(self) -> dict[str, Any | None]: "password": self.get_env_var_format("password"), } + if "schema" in self.profile_args: + profile["schema"] = self.get_env_var_format("schema") + return self.filter_null(profile) @property From 0185d8b3b898a26d48749dd8edcf8ee1a2af0274 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 21:16:13 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20f?= =?UTF-8?q?ormat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cosmos/profiles/postgres/user_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index b0c785d20..265604a8f 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -48,7 +48,7 @@ def profile(self) -> dict[str, Any | None]: if "schema" in self.profile_args: profile["schema"] = self.get_env_var_format("schema") - + return self.filter_null(profile) @property From 7c6d4a496a74f9ae602cab7cccd1eec78046bfad Mon Sep 17 00:00:00 2001 From: FouziaTariq <86288319+FouziaTariq@users.noreply.github.com> Date: Sun, 26 Nov 2023 02:00:58 +0100 Subject: [PATCH 07/10] Update user_pass.py --- cosmos/profiles/postgres/user_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 265604a8f..6520c8faf 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -47,7 +47,7 @@ def profile(self) -> dict[str, Any | None]: } if "schema" in self.profile_args: - profile["schema"] = self.get_env_var_format("schema") + profile["schema"] = self.profile_args["schema"] return self.filter_null(profile) From af00ce4b65e7b4c809dccdfa7bae2d2d849094bd Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 29 Feb 2024 13:14:45 +0000 Subject: [PATCH 08/10] Fix issue after making PostgresUserPasswordProfileMapping schema optional --- cosmos/profiles/postgres/user_pass.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index 6520c8faf..a081ff81a 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -54,9 +54,11 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: "Gets mock profile. Defaults port to 5432." - parent_mock = super().mock_profile - - return { + profile_dict = { "port": 5432, - **parent_mock, + **super().mock_profile, } + user_defined_schema = self.profile_args.get("schema") + if user_defined_schema: + profile_dict["schema"] = user_defined_schema + return profile_dict From e424454e14ddc4d33ba36465af1d465cf2ce3065 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 29 Feb 2024 13:15:48 +0000 Subject: [PATCH 09/10] Fix sources test --- tests/dbt/test_graph.py | 2 +- tests/sample/manifest_source.json | 116 ++++++++++++++++++++++++++---- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 4a0d4c98e..6044a64d0 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -457,7 +457,7 @@ def test_load_via_dbt_ls_with_sources(load_method): ), ) getattr(dbt_graph, load_method)() - assert len(dbt_graph.nodes) == 4 + assert len(dbt_graph.nodes) == 5 assert "source.simple.main.movies_ratings" in dbt_graph.nodes assert "exposure.simple.weekly_metrics" in dbt_graph.nodes diff --git a/tests/sample/manifest_source.json b/tests/sample/manifest_source.json index 9a84180f5..67f57035e 100644 --- a/tests/sample/manifest_source.json +++ b/tests/sample/manifest_source.json @@ -8,8 +8,10 @@ "exposure.simple.weekly_metrics" ], "source.simple.main.movies_ratings": [ - "model.simple.movies_ratings_simplified" - ] + "model.simple.movies_ratings_simplified", + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4" + ], + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": [] }, "disabled": {}, "docs": { @@ -28,7 +30,7 @@ "config": { "enabled": true }, - "created_at": 1696859933.549042, + "created_at": 1697205180.995924, "depends_on": { "macros": [], "nodes": [ @@ -1016,7 +1018,7 @@ "node_color": null, "show": true }, - "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "meta": {}, "name": "dates_in_range", "original_file_path": "macros/etc/datetime.sql", @@ -6945,8 +6947,8 @@ "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v8.json", "dbt_version": "1.4.0", "env": {}, - "generated_at": "2023-10-09T13:58:53.320926Z", - "invocation_id": "1790dc18-1177-4ca0-b993-a8eeb59f0c4c", + "generated_at": "2023-10-13T13:58:46.591195Z", + "invocation_id": "4f4b6d15-b4ab-4683-bbe5-efd94824b1d9", "project_id": "8dbdda48fb8748d6746f1965824e966a", "send_anonymous_usage_stats": true, "user_id": "4bdc3972-5c9f-4f16-90bd-3769a225fbe6" @@ -6986,7 +6988,7 @@ "tags": [], "unique_key": null }, - "created_at": 1696859933.5317938, + "created_at": 1697205180.9909241, "database": "database", "deferred": false, "depends_on": { @@ -7062,7 +7064,7 @@ "tags": [], "unique_key": null }, - "created_at": 1696859933.537527, + "created_at": 1697205180.984051, "database": "database", "deferred": false, "depends_on": { @@ -7103,6 +7105,84 @@ "unrendered_config": { "materialized": "table" } + }, + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": { + "alias": "source_not_null_imdb_movies_ratings_X", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "X", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "created_at": 1697205181.009168, + "database": "database", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "source.simple.main.movies_ratings" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "sources.imdb", + "fqn": [ + "simple", + "source_not_null_imdb_movies_ratings_X" + ], + "language": "sql", + "meta": {}, + "metrics": [], + "name": "source_not_null_imdb_movies_ratings_X", + "original_file_path": "models/source.yml", + "package_name": "simple", + "patch_path": null, + "path": "source_not_null_imdb_movies_ratings_X.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [], + "relation_name": null, + "resource_type": "test", + "schema": "main_dbt_test__audit", + "sources": [ + [ + "imdb", + "movies_ratings" + ] + ], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "X", + "model": "{{ get_where_subquery(source('imdb', 'movies_ratings')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4", + "unrendered_config": {} } }, "parent_map": { @@ -7115,18 +7195,30 @@ "model.simple.top_animations": [ "model.simple.movies_ratings_simplified" ], - "source.simple.main.movies_ratings": [] + "source.simple.main.movies_ratings": [], + "test.simple.source_not_null_imdb_movies_ratings_X.e684bf90f4": [ + "source.simple.main.movies_ratings" + ] }, "selectors": {}, "sources": { "source.simple.main.movies_ratings": { - "columns": {}, + "columns": { + "X": { + "data_type": null, + "description": "", + "meta": {}, + "name": "X", + "quote": null, + "tags": [] + } + }, "config": { "enabled": true }, - "created_at": 1696859933.549542, + "created_at": 1697205181.0098429, "database": "database", - "description": "Ratings by movie", + "description": "Ratings by movie\n", "external": null, "fqn": [ "simple", From 6069b1d23d8770d460174a9c8b1037eb8c3b2164 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 29 Feb 2024 13:26:17 +0000 Subject: [PATCH 10/10] Depending on the version of dbt, it leads to more or less nodes --- tests/dbt/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 6044a64d0..f72bbb146 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -457,7 +457,7 @@ def test_load_via_dbt_ls_with_sources(load_method): ), ) getattr(dbt_graph, load_method)() - assert len(dbt_graph.nodes) == 5 + assert len(dbt_graph.nodes) >= 4 assert "source.simple.main.movies_ratings" in dbt_graph.nodes assert "exposure.simple.weekly_metrics" in dbt_graph.nodes