From d255556d1ed4cd89b44b24eda56c7f36489c6e4d Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:00:32 +0530 Subject: [PATCH] Ensure compliance with linting rule D300 by using triple quotes for docstrings (#1049) Enable D300 to make docstring more consistent --- cosmos/config.py | 6 +++--- cosmos/dbt/selector.py | 2 +- cosmos/exceptions.py | 2 +- cosmos/operators/local.py | 8 ++++---- cosmos/profiles/__init__.py | 2 +- cosmos/profiles/athena/__init__.py | 2 +- cosmos/profiles/athena/access_key.py | 7 ++++--- cosmos/profiles/base.py | 6 +++--- cosmos/profiles/bigquery/__init__.py | 2 +- cosmos/profiles/bigquery/oauth.py | 7 ++++--- cosmos/profiles/bigquery/service_account_file.py | 7 ++++--- cosmos/profiles/bigquery/service_account_keyfile_dict.py | 7 ++++--- cosmos/profiles/databricks/__init__.py | 2 +- cosmos/profiles/databricks/token.py | 7 ++++--- cosmos/profiles/exasol/__init__.py | 2 +- cosmos/profiles/exasol/user_pass.py | 7 ++++--- cosmos/profiles/postgres/__init__.py | 2 +- cosmos/profiles/postgres/user_pass.py | 7 ++++--- cosmos/profiles/redshift/__init__.py | 2 +- cosmos/profiles/redshift/user_pass.py | 7 ++++--- cosmos/profiles/snowflake/__init__.py | 2 +- .../snowflake/user_encrypted_privatekey_env_variable.py | 7 ++++--- .../profiles/snowflake/user_encrypted_privatekey_file.py | 7 ++++--- cosmos/profiles/snowflake/user_pass.py | 7 ++++--- cosmos/profiles/snowflake/user_privatekey.py | 7 ++++--- cosmos/profiles/spark/__init__.py | 2 +- cosmos/profiles/spark/thrift.py | 3 ++- cosmos/profiles/trino/__init__.py | 2 +- cosmos/profiles/trino/base.py | 9 +++++---- cosmos/profiles/trino/certificate.py | 5 +++-- cosmos/profiles/trino/jwt.py | 5 +++-- cosmos/profiles/trino/ldap.py | 3 ++- cosmos/profiles/vertica/__init__.py | 2 +- cosmos/profiles/vertica/user_pass.py | 7 ++++--- dev/dags/dbt_docs.py | 2 +- docs/generate_mappings.py | 3 ++- pyproject.toml | 2 +- tests/profiles/athena/test_athena_access_key.py | 3 ++- tests/profiles/bigquery/test_bq_oauth.py | 2 +- tests/profiles/bigquery/test_bq_service_account_file.py | 2 +- tests/profiles/databricks/test_dbr_token.py | 2 +- tests/profiles/exasol/test_exasol_user_pass.py | 2 +- tests/profiles/postgres/test_pg_user_pass.py | 2 +- tests/profiles/redshift/test_redshift_user_pass.py | 2 +- ...t_snowflake_user_encrypted_privatekey_env_variable.py | 2 +- .../test_snowflake_user_encrypted_privatekey_file.py | 2 +- tests/profiles/snowflake/test_snowflake_user_pass.py | 2 +- .../profiles/snowflake/test_snowflake_user_privatekey.py | 2 +- tests/profiles/spark/test_spark_thrift.py | 2 +- tests/profiles/trino/test_trino_base.py | 2 +- tests/profiles/trino/test_trino_certificate.py | 2 +- tests/profiles/trino/test_trino_jwt.py | 2 +- tests/profiles/trino/test_trino_ldap.py | 2 +- tests/profiles/vertica/test_vertica_user_pass.py | 2 +- 54 files changed, 112 insertions(+), 92 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index 820833e6c..13622563e 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -244,7 +244,7 @@ def __post_init__(self) -> None: self.validate_profile() def validate_profile(self) -> None: - "Validates that we have enough information to render a profile." + """Validates that we have enough information to render a profile.""" if not self.profiles_yml_filepath and not self.profile_mapping: raise CosmosValueError("Either profiles_yml_filepath or profile_mapping must be set to render a profile") if self.profiles_yml_filepath and self.profile_mapping: @@ -253,7 +253,7 @@ def validate_profile(self) -> None: ) def validate_profiles_yml(self) -> None: - "Validates a user-supplied profiles.yml is present" + """Validates a user-supplied profiles.yml is present""" if self.profiles_yml_filepath and not Path(self.profiles_yml_filepath).exists(): raise CosmosValueError(f"The file {self.profiles_yml_filepath} does not exist.") @@ -261,7 +261,7 @@ def validate_profiles_yml(self) -> None: def ensure_profile( self, desired_profile_path: Path | None = None, use_mock_values: bool = False ) -> Iterator[tuple[Path, dict[str, str]]]: - "Context manager to ensure that there is a profile. If not, create one." + """Context manager to ensure that there is a profile. If not, create one.""" if self.profiles_yml_filepath: logger.info("Using user-supplied profiles.yml at %s", self.profiles_yml_filepath) yield Path(self.profiles_yml_filepath), {} diff --git a/cosmos/dbt/selector.py b/cosmos/dbt/selector.py index 47c118b28..257d60721 100644 --- a/cosmos/dbt/selector.py +++ b/cosmos/dbt/selector.py @@ -289,7 +289,7 @@ def select_nodes_ids_by_intersection(self) -> set[str]: return selected_nodes def _should_include_node(self, node_id: str, node: DbtNode) -> bool: - "Checks if a single node should be included. Only runs once per node with caching." + """Checks if a single node should be included. Only runs once per node with caching.""" logger.debug("Inspecting if the node <%s> should be included.", node_id) if node_id in self.visited_nodes: return node_id in self.selected_nodes diff --git a/cosmos/exceptions.py b/cosmos/exceptions.py index 85df285b1..308214475 100644 --- a/cosmos/exceptions.py +++ b/cosmos/exceptions.py @@ -1,4 +1,4 @@ -"Contains exceptions that Cosmos uses" +"""Contains exceptions that Cosmos uses""" class CosmosValueError(ValueError): diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 27d377916..4acaa9453 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -631,7 +631,7 @@ def __init__( folder_dir: str | None = None, **kwargs: Any, ) -> None: - "Initializes the operator." + """Initializes the operator.""" self.connection_id = connection_id self.bucket_name = bucket_name self.folder_dir = folder_dir @@ -674,7 +674,7 @@ def __init__( super().__init__(*args, **kwargs) def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to S3." + """Uploads the generated documentation to S3.""" logger.info( 'Attempting to upload generated docs to S3 using S3Hook("%s")', self.connection_id, @@ -740,7 +740,7 @@ def __init__( super().__init__(*args, **kwargs) def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to Azure Blob Storage." + """Uploads the generated documentation to Azure Blob Storage.""" logger.info( 'Attempting to upload generated docs to Azure Blob Storage using WasbHook(conn_id="%s")', self.connection_id, @@ -784,7 +784,7 @@ class DbtDocsGCSLocalOperator(DbtDocsCloudLocalOperator): ui_color = "#4772d5" def upload_to_cloud_storage(self, project_dir: str) -> None: - "Uploads the generated documentation to Google Cloud Storage" + """Uploads the generated documentation to Google Cloud Storage""" logger.info( 'Attempting to upload generated docs to Storage using GCSHook(conn_id="%s")', self.connection_id, diff --git a/cosmos/profiles/__init__.py b/cosmos/profiles/__init__.py index 5cc3109cc..b182bacf7 100644 --- a/cosmos/profiles/__init__.py +++ b/cosmos/profiles/__init__.py @@ -1,4 +1,4 @@ -"Contains a function to get the profile mapping based on the connection ID." +"""Contains a function to get the profile mapping based on the connection ID.""" from __future__ import annotations diff --git a/cosmos/profiles/athena/__init__.py b/cosmos/profiles/athena/__init__.py index 0cbb09a7c..3be305ace 100644 --- a/cosmos/profiles/athena/__init__.py +++ b/cosmos/profiles/athena/__init__.py @@ -1,4 +1,4 @@ -"Athena Airflow connection -> dbt profile mappings" +"""Athena Airflow connection -> dbt profile mappings""" from .access_key import AthenaAccessKeyProfileMapping diff --git a/cosmos/profiles/athena/access_key.py b/cosmos/profiles/athena/access_key.py index 8dc14f839..05c2ec099 100644 --- a/cosmos/profiles/athena/access_key.py +++ b/cosmos/profiles/athena/access_key.py @@ -1,4 +1,5 @@ -"Maps Airflow AWS connections to a dbt Athena profile using an access key id and secret access key." +"""Maps Airflow AWS connections to a dbt Athena profile using an access key id and secret access key.""" + from __future__ import annotations from typing import Any @@ -57,7 +58,7 @@ class AthenaAccessKeyProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" self.temporary_credentials = self._get_temporary_credentials() # type: ignore @@ -75,7 +76,7 @@ def profile(self) -> dict[str, Any | None]: @property def env_vars(self) -> dict[str, str]: - "Overwrites the env_vars for athena, Returns a dictionary of environment variables that should be set based on the self.temporary_credentials." + """Overwrites the env_vars for athena, Returns a dictionary of environment variables that should be set based on the self.temporary_credentials.""" if self.temporary_credentials is None: raise CosmosValueError(f"Could not find the athena credentials.") diff --git a/cosmos/profiles/base.py b/cosmos/profiles/base.py index d4b44b591..7c7b277b1 100755 --- a/cosmos/profiles/base.py +++ b/cosmos/profiles/base.py @@ -122,7 +122,7 @@ def _validate_disable_event_tracking(self) -> None: @property def conn(self) -> Connection: - "Returns the Airflow connection." + """Returns the Airflow connection.""" if not self._conn: conn = BaseHook.get_connection(self.conn_id) if not conn: @@ -197,7 +197,7 @@ def mock_profile(self) -> dict[str, Any]: @property def env_vars(self) -> dict[str, str]: - "Returns a dictionary of environment variables that should be set based on self.secret_fields." + """Returns a dictionary of environment variables that should be set based on self.secret_fields.""" env_vars = {} for field in self.secret_fields: @@ -287,7 +287,7 @@ def get_dbt_value(self, name: str) -> Any: @property def mapped_params(self) -> dict[str, Any]: - "Turns the self.airflow_param_mapping into a dictionary of dbt fields and their values." + """Turns the self.airflow_param_mapping into a dictionary of dbt fields and their values.""" mapped_params = { DBT_PROFILE_TYPE_FIELD: self.dbt_profile_type, } diff --git a/cosmos/profiles/bigquery/__init__.py b/cosmos/profiles/bigquery/__init__.py index b547bbf84..97217ea9d 100644 --- a/cosmos/profiles/bigquery/__init__.py +++ b/cosmos/profiles/bigquery/__init__.py @@ -1,4 +1,4 @@ -"BigQuery Airflow connection -> dbt profile mappings" +"""BigQuery Airflow connection -> dbt profile mappings""" from .oauth import GoogleCloudOauthProfileMapping from .service_account_file import GoogleCloudServiceAccountFileProfileMapping diff --git a/cosmos/profiles/bigquery/oauth.py b/cosmos/profiles/bigquery/oauth.py index 6ea02cfa4..f619a28ff 100644 --- a/cosmos/profiles/bigquery/oauth.py +++ b/cosmos/profiles/bigquery/oauth.py @@ -1,4 +1,5 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles that uses oauth via gcloud, if they don't use key file or JSON." +"""Maps Airflow GCP connections to dbt BigQuery profiles that uses oauth via gcloud, if they don't use key file or JSON.""" + from __future__ import annotations from typing import Any @@ -31,7 +32,7 @@ class GoogleCloudOauthProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. Defaults `threads` to 1." + """Generates profile. Defaults `threads` to 1.""" return { **self.mapped_params, "method": "oauth", @@ -41,7 +42,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return { diff --git a/cosmos/profiles/bigquery/service_account_file.py b/cosmos/profiles/bigquery/service_account_file.py index 02d00ffae..fb294a167 100644 --- a/cosmos/profiles/bigquery/service_account_file.py +++ b/cosmos/profiles/bigquery/service_account_file.py @@ -1,4 +1,5 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account file." +"""Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account file.""" + from __future__ import annotations from typing import Any @@ -32,7 +33,7 @@ class GoogleCloudServiceAccountFileProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. Defaults `threads` to 1." + """Generates profile. Defaults `threads` to 1.""" return { **self.mapped_params, "threads": 1, @@ -41,7 +42,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return { diff --git a/cosmos/profiles/bigquery/service_account_keyfile_dict.py b/cosmos/profiles/bigquery/service_account_keyfile_dict.py index 480db669b..cfa57056b 100644 --- a/cosmos/profiles/bigquery/service_account_keyfile_dict.py +++ b/cosmos/profiles/bigquery/service_account_keyfile_dict.py @@ -1,4 +1,5 @@ -"Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account keyfile dict/json." +"""Maps Airflow GCP connections to dbt BigQuery profiles if they use a service account keyfile dict/json.""" + from __future__ import annotations import json @@ -57,7 +58,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Generates mock profile. Defaults `threads` to 1." + """Generates mock profile. Defaults `threads` to 1.""" parent_mock_profile = super().mock_profile return {**parent_mock_profile, "threads": 1, "keyfile_json": None} @@ -86,5 +87,5 @@ def transform_keyfile_json(self, keyfile_json: str | dict[str, str]) -> dict[str @property def env_vars(self) -> dict[str, str]: - "Returns a dictionary of environment variables that should be set based on self.secret_fields." + """Returns a dictionary of environment variables that should be set based on self.secret_fields.""" return self._env_vars diff --git a/cosmos/profiles/databricks/__init__.py b/cosmos/profiles/databricks/__init__.py index dda537ab1..2e3a9d114 100644 --- a/cosmos/profiles/databricks/__init__.py +++ b/cosmos/profiles/databricks/__init__.py @@ -1,4 +1,4 @@ -"Databricks Airflow connection -> dbt profile mappings" +"""Databricks Airflow connection -> dbt profile mappings""" from .token import DatabricksTokenProfileMapping diff --git a/cosmos/profiles/databricks/token.py b/cosmos/profiles/databricks/token.py index cf7371302..78d97eb77 100644 --- a/cosmos/profiles/databricks/token.py +++ b/cosmos/profiles/databricks/token.py @@ -1,4 +1,5 @@ -"Maps Airflow Databricks connections with a token to dbt profiles." +"""Maps Airflow Databricks connections with a token to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -37,7 +38,7 @@ class DatabricksTokenProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Generates profile. The token is stored in an environment variable." + """Generates profile. The token is stored in an environment variable.""" return { **self.mapped_params, **self.profile_args, @@ -46,5 +47,5 @@ def profile(self) -> dict[str, Any | None]: } def transform_host(self, host: str) -> str: - "Removes the https:// prefix." + """Removes the https:// prefix.""" return host.replace("https://", "") diff --git a/cosmos/profiles/exasol/__init__.py b/cosmos/profiles/exasol/__init__.py index 48585d7fc..17062569d 100644 --- a/cosmos/profiles/exasol/__init__.py +++ b/cosmos/profiles/exasol/__init__.py @@ -1,4 +1,4 @@ -"Exasol Airflow connection -> dbt profile mappings" +"""Exasol Airflow connection -> dbt profile mappings""" from .user_pass import ExasolUserPasswordProfileMapping diff --git a/cosmos/profiles/exasol/user_pass.py b/cosmos/profiles/exasol/user_pass.py index 62311221d..aa5d66ebf 100644 --- a/cosmos/profiles/exasol/user_pass.py +++ b/cosmos/profiles/exasol/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Exasol connections with a username and password to dbt profiles." +"""Maps Airflow Exasol connections with a username and password to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -45,7 +46,7 @@ class ExasolUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -57,7 +58,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_dsn(self, host: str) -> str: - "Adds the port if it's not already there." + """Adds the port if it's not already there.""" if ":" not in host: port = self.conn.port or self.default_port return f"{host}:{port}" diff --git a/cosmos/profiles/postgres/__init__.py b/cosmos/profiles/postgres/__init__.py index 3b9a6c895..2c07cfd10 100644 --- a/cosmos/profiles/postgres/__init__.py +++ b/cosmos/profiles/postgres/__init__.py @@ -1,4 +1,4 @@ -"Postgres Airflow connection -> dbt profile mappings" +"""Postgres Airflow connection -> dbt profile mappings""" from .user_pass import PostgresUserPasswordProfileMapping diff --git a/cosmos/profiles/postgres/user_pass.py b/cosmos/profiles/postgres/user_pass.py index a081ff81a..c204ff8d4 100644 --- a/cosmos/profiles/postgres/user_pass.py +++ b/cosmos/profiles/postgres/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Postgres connections using user + password authentication to dbt profiles." +"""Maps Airflow Postgres connections using user + password authentication to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -37,7 +38,7 @@ class PostgresUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile = { "port": 5432, **self.mapped_params, @@ -53,7 +54,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5432." + """Gets mock profile. Defaults port to 5432.""" profile_dict = { "port": 5432, **super().mock_profile, diff --git a/cosmos/profiles/redshift/__init__.py b/cosmos/profiles/redshift/__init__.py index 3528e058d..c434850dc 100644 --- a/cosmos/profiles/redshift/__init__.py +++ b/cosmos/profiles/redshift/__init__.py @@ -1,4 +1,4 @@ -"Redshift Airflow connection -> dbt profile mappings" +"""Redshift Airflow connection -> dbt profile mappings""" from .user_pass import RedshiftUserPasswordProfileMapping diff --git a/cosmos/profiles/redshift/user_pass.py b/cosmos/profiles/redshift/user_pass.py index 0dd3e115f..adf89c2c6 100644 --- a/cosmos/profiles/redshift/user_pass.py +++ b/cosmos/profiles/redshift/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Redshift connections to dbt Redshift profiles if they use a username and password." +"""Maps Airflow Redshift connections to dbt Redshift profiles if they use a username and password.""" + from __future__ import annotations from typing import Any @@ -39,7 +40,7 @@ class RedshiftUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile = { "port": 5439, **self.mapped_params, @@ -52,7 +53,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5439." + """Gets mock profile. Defaults port to 5439.""" parent_mock = super().mock_profile return { diff --git a/cosmos/profiles/snowflake/__init__.py b/cosmos/profiles/snowflake/__init__.py index 01745d01e..7e81a96c7 100644 --- a/cosmos/profiles/snowflake/__init__.py +++ b/cosmos/profiles/snowflake/__init__.py @@ -1,4 +1,4 @@ -"Snowflake Airflow connection -> dbt profile mapping." +"""Snowflake Airflow connection -> dbt profile mapping.""" from .user_encrypted_privatekey_env_variable import SnowflakeEncryptedPrivateKeyPemProfileMapping from .user_encrypted_privatekey_file import SnowflakeEncryptedPrivateKeyFilePemProfileMapping diff --git a/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py b/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py index fecfa97fe..70722dd59 100644 --- a/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py +++ b/cosmos/profiles/snowflake/user_encrypted_privatekey_env_variable.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/private key." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key.""" + from __future__ import annotations import json @@ -73,7 +74,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -85,7 +86,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py b/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py index 6831cbd28..e217a6c22 100644 --- a/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py +++ b/cosmos/profiles/snowflake/user_encrypted_privatekey_file.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/private key path." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key path.""" + from __future__ import annotations import json @@ -72,7 +73,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -84,7 +85,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/snowflake/user_pass.py b/cosmos/profiles/snowflake/user_pass.py index fa634d1a2..a6042495e 100644 --- a/cosmos/profiles/snowflake/user_pass.py +++ b/cosmos/profiles/snowflake/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/password." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/password.""" + from __future__ import annotations import json @@ -72,7 +73,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -84,7 +85,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/snowflake/user_privatekey.py b/cosmos/profiles/snowflake/user_privatekey.py index 3aa0454ec..c74194b7a 100644 --- a/cosmos/profiles/snowflake/user_privatekey.py +++ b/cosmos/profiles/snowflake/user_privatekey.py @@ -1,4 +1,5 @@ -"Maps Airflow Snowflake connections to dbt profiles if they use a user/private key." +"""Maps Airflow Snowflake connections to dbt profiles if they use a user/private key.""" + from __future__ import annotations import json @@ -63,7 +64,7 @@ def conn(self) -> Connection: @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -75,7 +76,7 @@ def profile(self) -> dict[str, Any | None]: return self.filter_null(profile_vars) def transform_account(self, account: str) -> str: - "Transform the account to the format . if it's not already." + """Transform the account to the format . if it's not already.""" region = self.conn.extra_dejson.get("region") if region and region not in account: account = f"{account}.{region}" diff --git a/cosmos/profiles/spark/__init__.py b/cosmos/profiles/spark/__init__.py index d90b71dcc..9bbfd775a 100644 --- a/cosmos/profiles/spark/__init__.py +++ b/cosmos/profiles/spark/__init__.py @@ -1,4 +1,4 @@ -"Spark Airflow connection -> dbt profile mappings" +"""Spark Airflow connection -> dbt profile mappings""" from .thrift import SparkThriftProfileMapping diff --git a/cosmos/profiles/spark/thrift.py b/cosmos/profiles/spark/thrift.py index 3f9b44c04..e81851a74 100644 --- a/cosmos/profiles/spark/thrift.py +++ b/cosmos/profiles/spark/thrift.py @@ -1,4 +1,5 @@ -"Maps Airflow Spark connections to dbt profiles if they use a thrift connection." +"""Maps Airflow Spark connections to dbt profiles if they use a thrift connection.""" + from __future__ import annotations from typing import Any diff --git a/cosmos/profiles/trino/__init__.py b/cosmos/profiles/trino/__init__.py index adb633208..97431e450 100644 --- a/cosmos/profiles/trino/__init__.py +++ b/cosmos/profiles/trino/__init__.py @@ -1,4 +1,4 @@ -"Trino Airflow connection -> dbt profile mappings" +"""Trino Airflow connection -> dbt profile mappings""" from .base import TrinoBaseProfileMapping from .certificate import TrinoCertificateProfileMapping diff --git a/cosmos/profiles/trino/base.py b/cosmos/profiles/trino/base.py index 3ed5c7e6a..a3c242d8f 100644 --- a/cosmos/profiles/trino/base.py +++ b/cosmos/profiles/trino/base.py @@ -1,4 +1,5 @@ -"Maps common fields for Airflow Trino connections to dbt profiles." +"""Maps common fields for Airflow Trino connections to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -7,7 +8,7 @@ class TrinoBaseProfileMapping(BaseProfileMapping): - "Maps common fields for Airflow Trino connections to dbt profiles." + """Maps common fields for Airflow Trino connections to dbt profiles.""" airflow_connection_type: str = "trino" dbt_profile_type: str = "trino" @@ -31,7 +32,7 @@ class TrinoBaseProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any]: - "Gets profile." + """Gets profile.""" profile_vars = { **self.mapped_params, **self.profile_args, @@ -41,5 +42,5 @@ def profile(self) -> dict[str, Any]: return self.filter_null(profile_vars) def transform_host(self, host: str) -> str: - "Replaces http:// or https:// with nothing." + """Replaces http:// or https:// with nothing.""" return host.replace("http://", "").replace("https://", "") diff --git a/cosmos/profiles/trino/certificate.py b/cosmos/profiles/trino/certificate.py index c87e7c67c..1ee568738 100644 --- a/cosmos/profiles/trino/certificate.py +++ b/cosmos/profiles/trino/certificate.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to Certificate Trino dbt profiles." +"""Maps Airflow Trino connections to Certificate Trino dbt profiles.""" + from __future__ import annotations from typing import Any @@ -28,7 +29,7 @@ class TrinoCertificateProfileMapping(TrinoBaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" common_profile_vars = super().profile profile_vars = { **self.mapped_params, diff --git a/cosmos/profiles/trino/jwt.py b/cosmos/profiles/trino/jwt.py index 4f09cd3f1..f9c0fd60d 100644 --- a/cosmos/profiles/trino/jwt.py +++ b/cosmos/profiles/trino/jwt.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to JWT Trino dbt profiles." +"""Maps Airflow Trino connections to JWT Trino dbt profiles.""" + from __future__ import annotations from typing import Any @@ -29,7 +30,7 @@ class TrinoJWTProfileMapping(TrinoBaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile." + """Gets profile.""" common_profile_vars: dict[str, Any] = super().profile # need to remove jwt from profile_args because it will be set as an environment variable diff --git a/cosmos/profiles/trino/ldap.py b/cosmos/profiles/trino/ldap.py index 5ba122b93..7b1e7579b 100644 --- a/cosmos/profiles/trino/ldap.py +++ b/cosmos/profiles/trino/ldap.py @@ -1,4 +1,5 @@ -"Maps Airflow Trino connections to LDAP Trino dbt profiles." +"""Maps Airflow Trino connections to LDAP Trino dbt profiles.""" + from __future__ import annotations from typing import Any diff --git a/cosmos/profiles/vertica/__init__.py b/cosmos/profiles/vertica/__init__.py index 4a88f2edd..7a8a8acfd 100644 --- a/cosmos/profiles/vertica/__init__.py +++ b/cosmos/profiles/vertica/__init__.py @@ -1,4 +1,4 @@ -"Vertica Airflow connection -> dbt profile mappings" +"""Vertica Airflow connection -> dbt profile mappings""" from .user_pass import VerticaUserPasswordProfileMapping diff --git a/cosmos/profiles/vertica/user_pass.py b/cosmos/profiles/vertica/user_pass.py index e016b612c..214dbc357 100644 --- a/cosmos/profiles/vertica/user_pass.py +++ b/cosmos/profiles/vertica/user_pass.py @@ -1,4 +1,5 @@ -"Maps Airflow Vertica connections using username + password authentication to dbt profiles." +"""Maps Airflow Vertica connections using username + password authentication to dbt profiles.""" + from __future__ import annotations from typing import Any @@ -59,7 +60,7 @@ class VerticaUserPasswordProfileMapping(BaseProfileMapping): @property def profile(self) -> dict[str, Any | None]: - "Gets profile. The password is stored in an environment variable." + """Gets profile. The password is stored in an environment variable.""" profile = { "port": 5433, **self.mapped_params, @@ -72,7 +73,7 @@ def profile(self) -> dict[str, Any | None]: @property def mock_profile(self) -> dict[str, Any | None]: - "Gets mock profile. Defaults port to 5433." + """Gets mock profile. Defaults port to 5433.""" parent_mock = super().mock_profile return { diff --git a/dev/dags/dbt_docs.py b/dev/dags/dbt_docs.py index 924928a80..a7703d4e4 100644 --- a/dev/dags/dbt_docs.py +++ b/dev/dags/dbt_docs.py @@ -43,7 +43,7 @@ @task.branch(task_id="which_upload") def which_upload(): - "Only run the docs tasks if we have the proper connections set up" + """Only run the docs tasks if we have the proper connections set up""" downstream_tasks_to_run = [] try: diff --git a/docs/generate_mappings.py b/docs/generate_mappings.py index 8955efc73..049eb8b4e 100644 --- a/docs/generate_mappings.py +++ b/docs/generate_mappings.py @@ -15,7 +15,8 @@ @dataclass class Field: - "Represents a field in a profile mapping." + """Represents a field in a profile mapping.""" + dbt_name: str required: bool = False airflow_name: str | list[str] | None = None diff --git a/pyproject.toml b/pyproject.toml index c91b6abb3..c8bee0b20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,7 +208,7 @@ no_warn_unused_ignores = true [tool.ruff] line-length = 120 [tool.ruff.lint] -select = ["C901", "I", "F"] +select = ["C901", "D300", "I", "F"] ignore = ["F541"] [tool.ruff.lint.mccabe] max-complexity = 10 diff --git a/tests/profiles/athena/test_athena_access_key.py b/tests/profiles/athena/test_athena_access_key.py index c0a25b7e9..a59a6b652 100644 --- a/tests/profiles/athena/test_athena_access_key.py +++ b/tests/profiles/athena/test_athena_access_key.py @@ -1,4 +1,5 @@ -"Tests for the Athena profile." +"""Tests for the Athena profile.""" + from __future__ import annotations import json diff --git a/tests/profiles/bigquery/test_bq_oauth.py b/tests/profiles/bigquery/test_bq_oauth.py index f225f585f..d48cb8cc4 100644 --- a/tests/profiles/bigquery/test_bq_oauth.py +++ b/tests/profiles/bigquery/test_bq_oauth.py @@ -1,4 +1,4 @@ -"Tests for the BigQuery profile." +"""Tests for the BigQuery profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/bigquery/test_bq_service_account_file.py b/tests/profiles/bigquery/test_bq_service_account_file.py index 7b4cbd6ef..7c685b50b 100644 --- a/tests/profiles/bigquery/test_bq_service_account_file.py +++ b/tests/profiles/bigquery/test_bq_service_account_file.py @@ -1,4 +1,4 @@ -"Tests for the BigQuery profile." +"""Tests for the BigQuery profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/databricks/test_dbr_token.py b/tests/profiles/databricks/test_dbr_token.py index 0a1701af2..ada72f0e7 100644 --- a/tests/profiles/databricks/test_dbr_token.py +++ b/tests/profiles/databricks/test_dbr_token.py @@ -1,4 +1,4 @@ -"Tests for the postgres profile." +"""Tests for the postgres profile.""" from unittest.mock import patch diff --git a/tests/profiles/exasol/test_exasol_user_pass.py b/tests/profiles/exasol/test_exasol_user_pass.py index b2880c222..b4f4d14b4 100644 --- a/tests/profiles/exasol/test_exasol_user_pass.py +++ b/tests/profiles/exasol/test_exasol_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Exasol profile." +"""Tests for the Exasol profile.""" from unittest.mock import patch diff --git a/tests/profiles/postgres/test_pg_user_pass.py b/tests/profiles/postgres/test_pg_user_pass.py index 174fff139..c23e6add6 100644 --- a/tests/profiles/postgres/test_pg_user_pass.py +++ b/tests/profiles/postgres/test_pg_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the postgres profile." +"""Tests for the postgres profile.""" from unittest.mock import patch diff --git a/tests/profiles/redshift/test_redshift_user_pass.py b/tests/profiles/redshift/test_redshift_user_pass.py index 073ce2dee..f1e87b3cd 100644 --- a/tests/profiles/redshift/test_redshift_user_pass.py +++ b/tests/profiles/redshift/test_redshift_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Redshift profile." +"""Tests for the Redshift profile.""" from unittest.mock import patch diff --git a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py index 2c7515f72..ff00ee698 100644 --- a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py +++ b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_env_variable.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/private key environmentvariable profile." +"""Tests for the Snowflake user/private key environmentvariable profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py index d8c3aedcf..73f2d947d 100644 --- a/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py +++ b/tests/profiles/snowflake/test_snowflake_user_encrypted_privatekey_file.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/private key file profile." +"""Tests for the Snowflake user/private key file profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/snowflake/test_snowflake_user_pass.py b/tests/profiles/snowflake/test_snowflake_user_pass.py index 6276a20f1..8113d8528 100644 --- a/tests/profiles/snowflake/test_snowflake_user_pass.py +++ b/tests/profiles/snowflake/test_snowflake_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/password profile." +"""Tests for the Snowflake user/password profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/snowflake/test_snowflake_user_privatekey.py b/tests/profiles/snowflake/test_snowflake_user_privatekey.py index 2692792cf..edbf2b464 100644 --- a/tests/profiles/snowflake/test_snowflake_user_privatekey.py +++ b/tests/profiles/snowflake/test_snowflake_user_privatekey.py @@ -1,4 +1,4 @@ -"Tests for the Snowflake user/private key profile." +"""Tests for the Snowflake user/private key profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/spark/test_spark_thrift.py b/tests/profiles/spark/test_spark_thrift.py index da733e32f..2ac8303ca 100644 --- a/tests/profiles/spark/test_spark_thrift.py +++ b/tests/profiles/spark/test_spark_thrift.py @@ -1,4 +1,4 @@ -"Tests for the Spark profile." +"""Tests for the Spark profile.""" from unittest.mock import patch diff --git a/tests/profiles/trino/test_trino_base.py b/tests/profiles/trino/test_trino_base.py index ad1e84748..ee03821e2 100644 --- a/tests/profiles/trino/test_trino_base.py +++ b/tests/profiles/trino/test_trino_base.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/trino/test_trino_certificate.py b/tests/profiles/trino/test_trino_certificate.py index 4ab3589e8..81728c32d 100644 --- a/tests/profiles/trino/test_trino_certificate.py +++ b/tests/profiles/trino/test_trino_certificate.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/trino/test_trino_jwt.py b/tests/profiles/trino/test_trino_jwt.py index 36d2d2153..b886120f2 100644 --- a/tests/profiles/trino/test_trino_jwt.py +++ b/tests/profiles/trino/test_trino_jwt.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" import json from unittest.mock import patch diff --git a/tests/profiles/trino/test_trino_ldap.py b/tests/profiles/trino/test_trino_ldap.py index f8adddf93..98bb2a642 100644 --- a/tests/profiles/trino/test_trino_ldap.py +++ b/tests/profiles/trino/test_trino_ldap.py @@ -1,4 +1,4 @@ -"Tests for the Trino profile." +"""Tests for the Trino profile.""" from unittest.mock import patch diff --git a/tests/profiles/vertica/test_vertica_user_pass.py b/tests/profiles/vertica/test_vertica_user_pass.py index 6459dea96..cae259dff 100644 --- a/tests/profiles/vertica/test_vertica_user_pass.py +++ b/tests/profiles/vertica/test_vertica_user_pass.py @@ -1,4 +1,4 @@ -"Tests for the vertica profile." +"""Tests for the vertica profile.""" from unittest.mock import patch