From fd6f6f084f94271aa12649f7f19864f49ad867b7 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 10:46:42 +0200 Subject: [PATCH 01/15] Add Github action for integration test --- .github/workflows/integration.yml | 86 +++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 000000000..75203a1cb --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,86 @@ +# **what?** +# Runs integration tests. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to a release +# branch, and when manually triggered. + +name: Integration tests + +on: + push: + branches: + - "main" + - "*.latest" + - "releases/*" + pull_request: + workflow_dispatch: + +# explicitly turn off permissions for `GITHUB_TOKEN` +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + tests: + name: test with python ${{ matrix.python-version }} + + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + fail-fast: false + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + + env: + TOXENV: "unit" + PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv" + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: isbang/compose-action@v1.5.1 + with: + compose-file: "./docker/docker-compose.yml" + + - name: Install tox + run: | + python -m pip install --upgrade pip + python -m pip install tox + + - name: Run tox + run: | + tox -e integration-spark-session + tox -e integration-spark-thrift + + - name: Get current date + if: always() + id: date + run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT # Colons are not allowed in artifacts name + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: tests_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv + path: tests_results.csv From 795e40a01cfb1de47168eb0c8d49c231989d2e08 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 11:54:41 +0200 Subject: [PATCH 02/15] Update tox --- tox.ini | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 97017a926..e456d55d0 100644 --- a/tox.ini +++ b/tox.ini @@ -56,10 +56,7 @@ deps = [testenv:integration-spark-thrift] -allowlist_externals = - /bin/bash -basepython = python3.8 -commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*' +description = run integration tests against a Spark thrift server passenv = DBT_* PYTEST_ADDOPTS @@ -67,12 +64,10 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/dev-requirements.txt -e. +commands = pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/* [testenv:integration-spark-session] -allowlist_externals = - /bin/bash -basepython = python3.10 -commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*' +description = run integration tests against a Spark session passenv = DBT_* PYTEST_* @@ -81,3 +76,4 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/dev-requirements.txt -e.[session] +commands = pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/* From ff39c5d065e8b8ec065e5531e29107e35ccfcd6e Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 14:42:11 +0200 Subject: [PATCH 03/15] Fetch spark from https link --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bb4d378ed..b310fde4d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}" RUN apt-get update && \ apt-get install -y wget netcat procps libpostgresql-jdbc-java && \ - wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ + wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \ mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \ From 1505fc6fb4d26245e18e65485e73407c867a3ef3 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 14:42:40 +0200 Subject: [PATCH 04/15] Use Spark version 3.1.2 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b310fde4d..d1fd5357f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ ARG OPENJDK_VERSION=8 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE -ARG SPARK_VERSION=3.3.2 +ARG SPARK_VERSION=3.1.2 ARG HADOOP_VERSION=3 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ From 44fe33f4bd233f508c59c527a69590de1ec5f463 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 14:50:13 +0200 Subject: [PATCH 05/15] Seperate running Spark session and thrift --- .github/workflows/integration.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 75203a1cb..d455e804b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -69,10 +69,11 @@ jobs: python -m pip install --upgrade pip python -m pip install tox - - name: Run tox - run: | - tox -e integration-spark-session - tox -e integration-spark-thrift + - name: Run tox for Spark session + run: tox -e integration-spark-session + + - name: Run tox for Spark thrift + run: tox -e integration-spark-thrift - name: Get current date if: always() From 2655631fa3b6db8a7515f11495710675bca0ba4e Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 14:51:40 +0200 Subject: [PATCH 06/15] Use Spark 3.1.2 and Hadoop 3.2 --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d1fd5357f..85d01ba8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE -ARG SPARK_VERSION=3.1.2 -ARG HADOOP_VERSION=3 +ARG SPARK_VERSION=3.1.3 +ARG HADOOP_VERSION=3.2 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ org.label-schema.build-date=$BUILD_DATE \ From 915f67e9203dfb891ad4a22f3db7f9251b19ab84 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 14:57:20 +0200 Subject: [PATCH 07/15] Reset tox.ini --- tox.ini | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index e456d55d0..33055a211 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,9 @@ deps = [testenv:integration-spark-thrift] description = run integration tests against a Spark thrift server +allowlist_externals = + /bin/bash +basepython = python3.8 passenv = DBT_* PYTEST_ADDOPTS @@ -64,10 +67,13 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/dev-requirements.txt -e. -commands = pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/* +commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*' [testenv:integration-spark-session] description = run integration tests against a Spark session +allowlist_externals = + /bin/bash +basepython = python3.10 passenv = DBT_* PYTEST_* @@ -76,4 +82,4 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/dev-requirements.txt -e.[session] -commands = pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/* +commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*' From f0ef215e1c8186cf4270e695ec8663a5d745d127 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 15:08:48 +0200 Subject: [PATCH 08/15] Remove base pythons in tox.ini --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 33055a211..31396b5ef 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,6 @@ deps = description = run integration tests against a Spark thrift server allowlist_externals = /bin/bash -basepython = python3.8 passenv = DBT_* PYTEST_ADDOPTS @@ -73,7 +72,6 @@ commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posarg description = run integration tests against a Spark session allowlist_externals = /bin/bash -basepython = python3.10 passenv = DBT_* PYTEST_* From e8457df87d636324aae416c4a8eea363779f0156 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 15:19:19 +0200 Subject: [PATCH 09/15] Fix reference to Docker compose file --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d455e804b..517815e27 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -62,7 +62,7 @@ jobs: - uses: isbang/compose-action@v1.5.1 with: - compose-file: "./docker/docker-compose.yml" + compose-file: "./docker-compose.yml" - name: Install tox run: | From 842466a2883efd3a13826410f1477a0ff84c5e8f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 15:42:11 +0200 Subject: [PATCH 10/15] Remove timeout --- .github/workflows/integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 517815e27..8eafa5c72 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -36,7 +36,6 @@ jobs: name: test with python ${{ matrix.python-version }} runs-on: ubuntu-latest - timeout-minutes: 10 strategy: fail-fast: false From 0738f2d0bcc5f30eab1cc92b4c82720ce99e3265 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 15:55:55 +0200 Subject: [PATCH 11/15] Remove artifact steps --- .github/workflows/integration.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 8eafa5c72..9f26bd2be 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -73,14 +73,3 @@ jobs: - name: Run tox for Spark thrift run: tox -e integration-spark-thrift - - - name: Get current date - if: always() - id: date - run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT # Colons are not allowed in artifacts name - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: tests_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: tests_results.csv From 277bef1a2a4368d54b2b1ce41b7894c51d4f7ef1 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 15:56:33 +0200 Subject: [PATCH 12/15] Bump Spark and Hadoop versions --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 85d01ba8a..a9b9e0a2c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE -ARG SPARK_VERSION=3.1.3 -ARG HADOOP_VERSION=3.2 +ARG SPARK_VERSION=3.4.1 +ARG HADOOP_VERSION=3 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ org.label-schema.build-date=$BUILD_DATE \ From 8d5853d3049c5e299ab7d824ab33fc374a9894ff Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 16:08:16 +0200 Subject: [PATCH 13/15] Reset Spark and Hadoop version --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a9b9e0a2c..85d01ba8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE -ARG SPARK_VERSION=3.4.1 -ARG HADOOP_VERSION=3 +ARG SPARK_VERSION=3.1.3 +ARG HADOOP_VERSION=3.2 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \ org.label-schema.build-date=$BUILD_DATE \ From 919528ab14dd731f9efa913d37b051bda8922e44 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 16:09:09 +0200 Subject: [PATCH 14/15] Update comment --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9f26bd2be..f4c34c5fb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,7 +2,7 @@ # Runs integration tests. # **why?** -# Ensure code for dbt meets a certain quality standard. +# Ensure code runs as expected. # **when?** # This will run for all PRs, when code is pushed to a release From 15e48fd3f1f8d421f7f079a20ca8ba5fd5995d69 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond <jczuurmond@protonmail.com> Date: Fri, 29 Sep 2023 16:12:25 +0200 Subject: [PATCH 15/15] Add changie --- .changes/unreleased/Under the Hood-20230929-161218.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230929-161218.yaml diff --git a/.changes/unreleased/Under the Hood-20230929-161218.yaml b/.changes/unreleased/Under the Hood-20230929-161218.yaml new file mode 100644 index 000000000..c82e8252e --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230929-161218.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add Github action for integration testing +time: 2023-09-29T16:12:18.968755+02:00 +custom: + Author: JCZuurmond + Issue: "719"