From 329488250e84875d195b39cadf22d8e263abe07b Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 25 Apr 2022 13:55:46 -0700 Subject: [PATCH] Merge precommit hooks into main (#106) * Attempt at fixing changelog. * Add tools and black formatter instructions. (#105) * add pre-commit hook logic * streamline pre-commit check recipe for github * run hooks over repo code and reformat (#104) Co-authored-by: Mila Page --- .bumpversion.cfg | 4 +- .flake8 | 12 +++++ .github/pull_request_template.md | 2 +- .github/workflows/jira-creation.yml | 2 +- .github/workflows/jira-label.yml | 3 +- .github/workflows/jira-transition.yml | 2 +- .github/workflows/main.yml | 25 ++++----- .github/workflows/stale.yml | 2 - .github/workflows/version-bump.yml | 20 +++---- .gitignore | 5 +- .pre-commit-config.yaml | 66 ++++++++++++++++++++++++ CHANGELOG.md | 31 ++++++++++- MANIFEST.in | 2 +- Makefile | 61 ++++++++++++++++++++++ dbt/adapters/redshift/__init__.py | 7 +-- dbt/adapters/redshift/__version__.py | 2 +- dbt/adapters/redshift/connections.py | 63 +++++++++++----------- dbt/adapters/redshift/impl.py | 14 ++--- dbt/include/redshift/macros/adapters.sql | 10 ++-- dev_requirements.txt | 3 ++ scripts/build-dist.sh | 2 +- setup.py | 55 +++++++++----------- tox.ini | 11 +--- 23 files changed, 274 insertions(+), 130 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml create mode 100644 Makefile diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cdb7cde23..5665b64ef 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -4,7 +4,7 @@ parse = (?P\d+) \.(?P\d+) \.(?P\d+) ((?Pa|b|rc)(?P\d+))? -serialize = +serialize = {major}.{minor}.{patch}{prerelease}{num} {major}.{minor}.{patch} commit = False @@ -13,7 +13,7 @@ tag = False [bumpversion:part:prerelease] first_value = a optional_value = final -values = +values = a b rc diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..f39d154c0 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +select = + E + W + F +ignore = + W503 # makes Flake8 work like black + W504 + E203 # makes Flake8 work like black + E741 + E501 +exclude = test diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5e7cc23d3..b99e90e57 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -18,4 +18,4 @@ resolves # - [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements) - [ ] I have run this code in development and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR -- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-redshift next" section. \ No newline at end of file +- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-redshift next" section. diff --git a/.github/workflows/jira-creation.yml b/.github/workflows/jira-creation.yml index c84e106a7..b4016befc 100644 --- a/.github/workflows/jira-creation.yml +++ b/.github/workflows/jira-creation.yml @@ -13,7 +13,7 @@ name: Jira Issue Creation on: issues: types: [opened, labeled] - + permissions: issues: write diff --git a/.github/workflows/jira-label.yml b/.github/workflows/jira-label.yml index fd533a170..3da2e3a38 100644 --- a/.github/workflows/jira-label.yml +++ b/.github/workflows/jira-label.yml @@ -13,7 +13,7 @@ name: Jira Label Mirroring on: issues: types: [labeled, unlabeled] - + permissions: issues: read @@ -24,4 +24,3 @@ jobs: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} - diff --git a/.github/workflows/jira-transition.yml b/.github/workflows/jira-transition.yml index 71273c7a9..ed9f9cd4f 100644 --- a/.github/workflows/jira-transition.yml +++ b/.github/workflows/jira-transition.yml @@ -21,4 +21,4 @@ jobs: secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} \ No newline at end of file + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ccdaf1aff..ea2641407 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,19 +37,10 @@ defaults: jobs: code-quality: - name: ${{ matrix.toxenv }} + name: code-quality runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - toxenv: [flake8] - - env: - TOXENV: ${{ matrix.toxenv }} - PYTEST_ADDOPTS: "-v --color=yes" - steps: - name: Check out the repository uses: actions/checkout@v2 @@ -62,12 +53,14 @@ jobs: - name: Install python dependencies run: | pip install --user --upgrade pip - pip install tox + pip install -r dev_requirements.txt pip --version - tox --version + pre-commit --version + mypy --version + dbt --version - - name: Run tox - run: tox + - name: pre-commit hooks + run: pre-commit run --all-files --show-diff-on-failure unit: name: unit test / python ${{ matrix.python-version }} @@ -119,7 +112,7 @@ jobs: name: build packages runs-on: ubuntu-latest - + outputs: is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} @@ -153,7 +146,7 @@ jobs: - name: Check wheel contents run: | check-wheel-contents dist/*.whl --ignore W007,W008 - + - name: Check if this is an alpha version id: check-is-alpha run: | diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2848ce8f7..a56455d55 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,5 +13,3 @@ jobs: stale-pr-message: "This PR has been marked as Stale because it has been open for 180 days with no activity. If you would like the PR to remain open, please remove the stale label or comment on the PR, or it will be closed in 7 days." # mark issues/PRs stale when they haven't seen activity in 180 days days-before-stale: 180 - # ignore checking issues with the following labels - exempt-issue-labels: "epic, discussion" \ No newline at end of file diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 4913a6e84..b0a3174df 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -1,16 +1,16 @@ # **what?** # This workflow will take a version number and a dry run flag. With that -# it will run versionbump to update the version number everywhere in the +# it will run versionbump to update the version number everywhere in the # code base and then generate an update Docker requirements file. If this # is a dry run, a draft PR will open with the changes. If this isn't a dry # run, the changes will be committed to the branch this is run on. # **why?** -# This is to aid in releasing dbt and making sure we have updated +# This is to aid in releasing dbt and making sure we have updated # the versions and Docker requirements in all places. # **when?** -# This is triggered either manually OR +# This is triggered either manually OR # from the repository_dispatch event "version-bump" which is sent from # the dbt-release repo Action @@ -25,11 +25,11 @@ on: is_dry_run: description: 'Creates a draft PR to allow testing instead of committing to a branch' required: true - default: 'true' + default: 'true' repository_dispatch: types: [version-bump] -jobs: +jobs: bump: runs-on: ubuntu-latest steps: @@ -57,19 +57,19 @@ jobs: run: | python3 -m venv env source env/bin/activate - pip install --upgrade pip - + pip install --upgrade pip + - name: Create PR branch if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }} run: | git checkout -b bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID git push origin bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID git branch --set-upstream-to=origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID - + - name: Bumping version run: | source env/bin/activate - pip install -r dev_requirements.txt + pip install -r dev_requirements.txt env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major git status @@ -99,4 +99,4 @@ jobs: draft: true base: ${{github.ref}} title: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}' - branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' + branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}' diff --git a/.gitignore b/.gitignore index 43724b61e..4e2f60b9f 100644 --- a/.gitignore +++ b/.gitignore @@ -50,8 +50,7 @@ coverage.xml .hypothesis/ test.env -# Mypy -.mypy_cache/ +*.pytest_cache/ # Translations *.mo @@ -78,6 +77,7 @@ target/ # Vim *.sw* +# Pyenv .python-version # Vim @@ -90,6 +90,7 @@ venv/ # AWS credentials .aws/ +# MacOS .DS_Store # vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..ccaa093bf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,66 @@ +# For more on configuring pre-commit hooks (see https://pre-commit.com/) + +# TODO: remove global exclusion of tests when testing overhaul is complete +exclude: '^tests/.*' + +# Force all unspecified python hooks to run python 3.8 +default_language_version: + python: python3.8 + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict +- repo: https://github.com/psf/black + rev: 21.12b0 + hooks: + - id: black + additional_dependencies: ['click==8.0.4'] + args: + - "--line-length=99" + - "--target-version=py38" + - id: black + alias: black-check + stages: [manual] + additional_dependencies: ['click==8.0.4'] + args: + - "--line-length=99" + - "--target-version=py38" + - "--check" + - "--diff" +- repo: https://gitlab.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + - id: flake8 + alias: flake8-check + stages: [manual] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.782 + hooks: + - id: mypy + # N.B.: Mypy is... a bit fragile. + # + # By using `language: system` we run this hook in the local + # environment instead of a pre-commit isolated one. This is needed + # to ensure mypy correctly parses the project. + + # It may cause trouble in that it adds environmental variables out + # of our control to the mix. Unfortunately, there's nothing we can + # do about per pre-commit's author. + # See https://github.com/pre-commit/pre-commit/issues/730 for details. + args: [--show-error-codes, --ignore-missing-imports] + files: ^dbt/adapters/.* + language: system + - id: mypy + alias: mypy-check + stages: [manual] + args: [--show-error-codes, --pretty, --ignore-missing-imports] + files: ^dbt/adapters + language: system diff --git a/CHANGELOG.md b/CHANGELOG.md index 931f0294f..f0a766c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,37 @@ ## dbt-redshift 1.1.0 (TBD) +### Under the hood +- Add precommits for this repo ([#XX](https://github.com/dbt-labs/dbt-snowflake/pull/XX)) + +## dbt-redshift 1.0.1 (April 19, 2022) + +### Fixes +- Fix table creation statement ordering when including both the BACKUP parameter as well as the dist/sort keys ([#60](https://github.com/dbt-labs/dbt-redshift/issues/60), [#63](https://github.com/dbt-labs/dbt-redshift/pull/63)) + +### Under the hood +- Install compatible branch of dbt-core in unit/integration tests based on merge target ([#80](https://github.com/dbt-labs/dbt-redshift/pull/80)) +- Fix test related to preventing coercion of boolean values (True,False) to numeric values (0,1) in query results ([#58](https://github.com/dbt-labs/dbt-redshift/pull/58)) + +### Contributors +- [@SMeltser](https://github.com/SMeltser) ([#63](https://github.com/dbt-labs/dbt-redshift/pull/63)) + +## dbt-redshift 1.1.0rc1 (April 13, 2022) + ### Under the hood - Use dbt.tests.adapter.basic in test suite ([#78](https://github.com/dbt-labs/dbt-redshift/issues/78), [#81](https://github.com/dbt-labs/dbt-redshift/pull/81)) +## dbt-redshift 1.0.1rc1 (April 6, 2022) + +### Fixes +- Fix table creation statement ordering when including both the BACKUP parameter as well as the dist/sort keys ([#60](https://github.com/dbt-labs/dbt-redshift/issues/60), [#63](https://github.com/dbt-labs/dbt-redshift/pull/63)) + +### Under the hood +- Install compatible branch of dbt-core in unit/integration tests based on merge target ([#80](https://github.com/dbt-labs/dbt-redshift/pull/80)) +- Fix test related to preventing coercion of boolean values (True,False) to numeric values (0,1) in query results ([#58](https://github.com/dbt-labs/dbt-redshift/pull/58)) + +### Contributors +- [@SMeltser](https://github.com/SMeltser) ([#63](https://github.com/dbt-labs/dbt-redshift/pull/63)) + ## dbt-redshift 1.1.0b1 (March 23, 2022) ### Fixes @@ -18,8 +47,6 @@ ### Contributors - [@SMeltser](https://github.com/SMeltser)([#63](https://github.com/dbt-labs/dbt-redshift/pull/63)) -## dbt-redshift 1.0.1 (TBD) - ## dbt-redshift 1.0.0 (December 3, 2021) ## dbt-redshift 1.0.0rc2 (November 24, 2021) diff --git a/MANIFEST.in b/MANIFEST.in index 78412d5b8..cfbc714ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include dbt/include *.sql *.yml *.md \ No newline at end of file +recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..05af9a2ca --- /dev/null +++ b/Makefile @@ -0,0 +1,61 @@ +.DEFAULT_GOAL:=help + +.PHONY: dev +dev: ## Installs adapter in develop mode along with development depedencies + @\ + pip install -r dev_requirements.txt && pre-commit install + +.PHONY: mypy +mypy: ## Runs mypy against staged changes for static type checking. + @\ + pre-commit run --hook-stage manual mypy-check | grep -v "INFO" + +.PHONY: flake8 +flake8: ## Runs flake8 against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual flake8-check | grep -v "INFO" + +.PHONY: black +black: ## Runs black against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual black-check -v | grep -v "INFO" + +.PHONY: lint +lint: ## Runs flake8 and mypy code checks against staged changes. + @\ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" + +.PHONY: linecheck +linecheck: ## Checks for all Python lines 100 characters or more + @\ + find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; + +.PHONY: unit +unit: ## Runs unit tests with py38. + @\ + tox -e py38 + +.PHONY: test +test: ## Runs unit tests with py38 and code checks against staged changes. + @\ + tox -p -e py38; \ + pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" + +.PHONY: integration +integration: ## Runs snowflake integration tests with py38. + @\ + tox -e py38-snowflake -- + +.PHONY: clean + @echo "cleaning repo" + @git clean -f -X + +.PHONY: help +help: ## Show this help message. + @echo 'usage: make [target]' + @echo + @echo 'targets:' + @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py index f8059e116..0d8e1cd10 100644 --- a/dbt/adapters/redshift/__init__.py +++ b/dbt/adapters/redshift/__init__.py @@ -5,11 +5,12 @@ from dbt.adapters.redshift.impl import RedshiftAdapter -from dbt.adapters.base import AdapterPlugin -from dbt.include import redshift +from dbt.adapters.base import AdapterPlugin # type: ignore +from dbt.include import redshift # type: ignore Plugin = AdapterPlugin( adapter=RedshiftAdapter, credentials=RedshiftCredentials, include_path=redshift.PACKAGE_PATH, - dependencies=['postgres']) + dependencies=["postgres"], +) diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 8fa663be6..a6b977228 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = '1.2.0a1' +version = "1.2.0a1" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index e0afc1ffc..70b94b10b 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -17,33 +17,33 @@ logger = AdapterLogger("Redshift") -drop_lock: Lock = dbt.flags.MP_CONTEXT.Lock() +drop_lock: Lock = dbt.flags.MP_CONTEXT.Lock() # type: ignore -IAMDuration = NewType('IAMDuration', int) +IAMDuration = NewType("IAMDuration", int) class IAMDurationEncoder(FieldEncoder): @property def json_schema(self): - return {'type': 'integer', 'minimum': 0, 'maximum': 65535} + return {"type": "integer", "minimum": 0, "maximum": 65535} dbtClassMixin.register_field_encoders({IAMDuration: IAMDurationEncoder()}) class RedshiftConnectionMethod(StrEnum): - DATABASE = 'database' - IAM = 'iam' + DATABASE = "database" + IAM = "iam" @dataclass class RedshiftCredentials(PostgresCredentials): - method: RedshiftConnectionMethod = RedshiftConnectionMethod.DATABASE + method: str = RedshiftConnectionMethod.DATABASE # type: ignore password: Optional[str] = None cluster_id: Optional[str] = field( default=None, - metadata={'description': 'If using IAM auth, the name of the cluster'}, + metadata={"description": "If using IAM auth, the name of the cluster"}, ) iam_profile: Optional[str] = None iam_duration_seconds: int = 900 @@ -55,20 +55,15 @@ class RedshiftCredentials(PostgresCredentials): @property def type(self): - return 'redshift' + return "redshift" def _connection_keys(self): keys = super()._connection_keys() - return keys + ( - 'method', - 'cluster_id', - 'iam_profile', - 'iam_duration_seconds' - ) + return keys + ("method", "cluster_id", "iam_profile", "iam_duration_seconds") class RedshiftConnectionManager(PostgresConnectionManager): - TYPE = 'redshift' + TYPE = "redshift" @contextmanager def fresh_transaction(self, name=None): @@ -94,9 +89,9 @@ def fresh_transaction(self, name=None): self.begin() @classmethod - def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, - iam_profile, duration_s, autocreate, - db_groups): + def fetch_cluster_credentials( + cls, db_user, db_name, cluster_id, iam_profile, duration_s, autocreate, db_groups + ): """Fetches temporary login credentials from AWS. The specified user must already exist in the database, or else an error will occur""" @@ -104,12 +99,9 @@ def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, session = boto3.Session() boto_client = session.client("redshift") else: - logger.debug("Connecting to Redshift using 'IAM'" + - f"with profile {iam_profile}") - boto_session = boto3.Session( - profile_name=iam_profile - ) - boto_client = boto_session.client('redshift') + logger.debug("Connecting to Redshift using 'IAM'" + f"with profile {iam_profile}") + boto_session = boto3.Session(profile_name=iam_profile) + boto_client = boto_session.client("redshift") try: return boto_client.get_cluster_credentials( @@ -118,12 +110,13 @@ def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, ClusterIdentifier=cluster_id, DurationSeconds=duration_s, AutoCreate=autocreate, - DbGroups=db_groups,) + DbGroups=db_groups, + ) except boto_client.exceptions.ClientError as e: raise dbt.exceptions.FailedToConnectException( - "Unable to get temporary Redshift cluster credentials: {}" - .format(e)) + "Unable to get temporary Redshift cluster credentials: {}".format(e) + ) @classmethod def get_tmp_iam_cluster_credentials(cls, credentials): @@ -135,8 +128,8 @@ def get_tmp_iam_cluster_credentials(cls, credentials): if not cluster_id: raise dbt.exceptions.FailedToConnectException( - "'cluster_id' must be provided in profile if IAM " - "authentication method selected") + "'cluster_id' must be provided in profile if IAM " "authentication method selected" + ) cluster_creds = cls.fetch_cluster_credentials( credentials.user, @@ -149,15 +142,16 @@ def get_tmp_iam_cluster_credentials(cls, credentials): ) # replace username and password with temporary redshift credentials - return credentials.replace(user=cluster_creds.get('DbUser'), - password=cluster_creds.get('DbPassword')) + return credentials.replace( + user=cluster_creds.get("DbUser"), password=cluster_creds.get("DbPassword") + ) @classmethod def get_credentials(cls, credentials): method = credentials.method # Support missing 'method' for backwards compatibility - if method == 'database' or method is None: + if method == "database" or method is None: logger.debug("Connecting to Redshift using 'database' credentials") # this requirement is really annoying to encode into json schema, # so validate it here @@ -167,10 +161,11 @@ def get_credentials(cls, credentials): ) return credentials - elif method == 'iam': + elif method == "iam": logger.debug("Connecting to Redshift using 'IAM' credentials") return cls.get_tmp_iam_cluster_credentials(credentials) else: raise dbt.exceptions.FailedToConnectException( - "Invalid 'method' in profile: '{}'".format(method)) + "Invalid 'method' in profile: '{}'".format(method) + ) diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index 0ee975973..c0b14ef0f 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -31,7 +31,7 @@ class RedshiftAdapter(PostgresAdapter, SQLAdapter): @classmethod def date_function(cls): - return 'getdate()' + return "getdate()" def drop_relation(self, relation): """ @@ -73,11 +73,12 @@ def verify_database(self, database): if database.lower() != expected.lower() and not ra3_node: raise dbt.exceptions.NotImplementedException( - 'Cross-db references allowed only in RA3.* node. ({} vs {})' - .format(database, expected) + "Cross-db references allowed only in RA3.* node. ({} vs {})".format( + database, expected + ) ) # return an empty string on success so macros can call this - return '' + return "" def _get_catalog_schemas(self, manifest): # redshift(besides ra3) only allow one database (the main one) @@ -86,6 +87,7 @@ def _get_catalog_schemas(self, manifest): return schemas.flatten(allow_multiple_databases=self.config.credentials.ra3_node) except dbt.exceptions.RuntimeException as exc: dbt.exceptions.raise_compiler_error( - 'Cross-db references allowed only in {} RA3.* node. Got {}' - .format(self.type(), exc.msg) + "Cross-db references allowed only in {} RA3.* node. Got {}".format( + self.type(), exc.msg + ) ) diff --git a/dbt/include/redshift/macros/adapters.sql b/dbt/include/redshift/macros/adapters.sql index e82931857..1644f884d 100644 --- a/dbt/include/redshift/macros/adapters.sql +++ b/dbt/include/redshift/macros/adapters.sql @@ -259,9 +259,9 @@ {% macro redshift__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} - + {% if add_columns %} - + {% for column in add_columns %} {% set sql -%} alter {{ relation.type }} {{ relation }} add column {{ column.name }} {{ column.data_type }} @@ -270,16 +270,16 @@ {% endfor %} {% endif %} - + {% if remove_columns %} - + {% for column in remove_columns %} {% set sql -%} alter {{ relation.type }} {{ relation }} drop column {{ column.name }} {% endset %} {% do run_query(sql) %} {% endfor %} - + {% endif %} {% endmacro %} diff --git a/dev_requirements.txt b/dev_requirements.txt index 1dc51f545..840cab5ff 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,6 +4,8 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres +black==21.12b0 +click~=8.0.4 bumpversion flake8 flaky @@ -11,6 +13,7 @@ freezegun==0.3.12 ipdb mypy==0.782 pip-tools +pre-commit pytest pytest-dotenv pytest-logbook diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh index 65e6dbc97..3c3808399 100755 --- a/scripts/build-dist.sh +++ b/scripts/build-dist.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -eo pipefail diff --git a/setup.py b/setup.py index 767911819..afb7db56d 100644 --- a/setup.py +++ b/setup.py @@ -5,41 +5,39 @@ # require python 3.7 or newer if sys.version_info < (3, 7): - print('Error: dbt does not support this version of Python.') - print('Please upgrade to Python 3.7 or higher.') + print("Error: dbt does not support this version of Python.") + print("Please upgrade to Python 3.7 or higher.") sys.exit(1) # require version of setuptools that supports find_namespace_packages from setuptools import setup + try: from setuptools import find_namespace_packages except ImportError: # the user has a downlevel version of setuptools. - print('Error: dbt requires setuptools v40.1.0 or higher.') - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' - 'and try again') + print("Error: dbt requires setuptools v40.1.0 or higher.") + print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") sys.exit(1) # pull long description from README this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: +with open(os.path.join(this_directory, "README.md")) as f: long_description = f.read() # get this package's version from dbt/adapters//__version__.py def _get_plugin_version_dict(): - _version_path = os.path.join( - this_directory, 'dbt', 'adapters', 'redshift', '__version__.py' - ) - _semver = r'''(?P\d+)\.(?P\d+)\.(?P\d+)''' - _pre = r'''((?Pa|b|rc)(?P
\d+))?'''
-    _version_pattern = fr'''version\s*=\s*["']{_semver}{_pre}["']'''
+    _version_path = os.path.join(this_directory, "dbt", "adapters", "redshift", "__version__.py")
+    _semver = r"""(?P\d+)\.(?P\d+)\.(?P\d+)"""
+    _pre = r"""((?Pa|b|rc)(?P
\d+))?"""
+    _version_pattern = fr"""version\s*=\s*["']{_semver}{_pre}["']"""
     with open(_version_path) as f:
         match = re.search(_version_pattern, f.read().strip())
         if match is None:
-            raise ValueError(f'invalid version at {_version_path}')
+            raise ValueError(f"invalid version at {_version_path}")
         return match.groupdict()
 
 
@@ -47,7 +45,7 @@ def _get_plugin_version_dict():
 def _get_dbt_core_version():
     parts = _get_plugin_version_dict()
     minor = "{major}.{minor}.0".format(**parts)
-    pre = (parts["prekind"]+"1" if parts["prekind"] else "")
+    pre = parts["prekind"] + "1" if parts["prekind"] else ""
     return f"{minor}{pre}"
 
 
@@ -61,31 +59,28 @@ def _get_dbt_core_version():
     version=package_version,
     description=description,
     long_description=long_description,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     author="dbt Labs",
     author_email="info@dbtlabs.com",
     url="https://github.com/dbt-labs/dbt-redshift",
-    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
+    packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        'dbt-core~={}'.format(dbt_core_version),
-        'dbt-postgres~={}'.format(dbt_core_version),
+        "dbt-core~={}".format(dbt_core_version),
+        "dbt-postgres~={}".format(dbt_core_version),
         # the following are all to match snowflake-connector-python
-        'boto3>=1.4.4,<2.0.0',
+        "boto3>=1.4.4,<2.0.0",
     ],
     zip_safe=False,
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
+        "Development Status :: 5 - Production/Stable",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: Microsoft :: Windows",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     python_requires=">=3.7",
 )
diff --git a/tox.ini b/tox.ini
index bc1404887..3aa00dfbe 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,15 +1,6 @@
 [tox]
 skipsdist = True
-envlist = py37,py38,py39,flake8
-
-[testenv:flake8]
-description = flake8 code checks
-basepython = python3.8
-skip_install = true
-commands = flake8 --select=E,W,F --ignore=W504,E741 --max-line-length 99 \
-  dbt
-deps =
-  -rdev_requirements.txt
+envlist = py37,py38,py39
 
 [testenv:{unit,py37,py38,py39,py}]
 description = unit testing