From fd6f6f084f94271aa12649f7f19864f49ad867b7 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 10:46:42 +0200
Subject: [PATCH 01/15] Add Github action for integration test

---
 .github/workflows/integration.yml | 86 +++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 .github/workflows/integration.yml

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 000000000..75203a1cb
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,86 @@
+# **what?**
+# Runs integration tests.
+
+# **why?**
+# Ensure code for dbt meets a certain quality standard.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+
+# explicitly turn off permissions for `GITHUB_TOKEN`
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  tests:
+    name: test with python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+
+    env:
+      TOXENV: "unit"
+      PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: isbang/compose-action@v1.5.1
+        with:
+          compose-file: "./docker/docker-compose.yml"
+
+      - name: Install tox
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox
+
+      - name: Run tox
+        run: |
+          tox -e integration-spark-session
+          tox -e integration-spark-thrift
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT  # Colons are not allowed in artifacts name
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: tests_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: tests_results.csv

From 795e40a01cfb1de47168eb0c8d49c231989d2e08 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 11:54:41 +0200
Subject: [PATCH 02/15] Update tox

---
 tox.ini | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/tox.ini b/tox.ini
index 97017a926..e456d55d0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,10 +56,7 @@ deps =
 
 
 [testenv:integration-spark-thrift]
-allowlist_externals =
-    /bin/bash
-basepython = python3.8
-commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*'
+description = run integration tests against a Spark thrift server
 passenv =
     DBT_*
     PYTEST_ADDOPTS
@@ -67,12 +64,10 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.
+commands = pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*
 
 [testenv:integration-spark-session]
-allowlist_externals =
-    /bin/bash
-basepython = python3.10
-commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*'
+description = run integration tests against a Spark session
 passenv =
     DBT_*
     PYTEST_*
@@ -81,3 +76,4 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.[session]
+commands = pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*

From ff39c5d065e8b8ec065e5531e29107e35ccfcd6e Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 14:42:11 +0200
Subject: [PATCH 03/15] Fetch spark from https link

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index bb4d378ed..b310fde4d 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -14,7 +14,7 @@ ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
 
 RUN apt-get update && \
     apt-get install -y wget netcat procps libpostgresql-jdbc-java && \
-    wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
+    wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \

From 1505fc6fb4d26245e18e65485e73407c867a3ef3 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 14:42:40 +0200
Subject: [PATCH 04/15] Use Spark version 3.1.2

---
 docker/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index b310fde4d..d1fd5357f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,7 +2,7 @@ ARG OPENJDK_VERSION=8
 FROM eclipse-temurin:${OPENJDK_VERSION}-jre
 
 ARG BUILD_DATE
-ARG SPARK_VERSION=3.3.2
+ARG SPARK_VERSION=3.1.2
 ARG HADOOP_VERSION=3
 
 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \

From 44fe33f4bd233f508c59c527a69590de1ec5f463 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 14:50:13 +0200
Subject: [PATCH 05/15] Seperate running Spark session and thrift

---
 .github/workflows/integration.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 75203a1cb..d455e804b 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -69,10 +69,11 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install tox
 
-      - name: Run tox
-        run: |
-          tox -e integration-spark-session
-          tox -e integration-spark-thrift
+      - name: Run tox for Spark session
+        run: tox -e integration-spark-session
+
+      - name: Run tox for Spark thrift
+        run: tox -e integration-spark-thrift
 
       - name: Get current date
         if: always()

From 2655631fa3b6db8a7515f11495710675bca0ba4e Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 14:51:40 +0200
Subject: [PATCH 06/15] Use Spark 3.1.2 and Hadoop 3.2

---
 docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index d1fd5357f..85d01ba8a 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8
 FROM eclipse-temurin:${OPENJDK_VERSION}-jre
 
 ARG BUILD_DATE
-ARG SPARK_VERSION=3.1.2
-ARG HADOOP_VERSION=3
+ARG SPARK_VERSION=3.1.3
+ARG HADOOP_VERSION=3.2
 
 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \
       org.label-schema.build-date=$BUILD_DATE \

From 915f67e9203dfb891ad4a22f3db7f9251b19ab84 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 14:57:20 +0200
Subject: [PATCH 07/15] Reset tox.ini

---
 tox.ini | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index e456d55d0..33055a211 100644
--- a/tox.ini
+++ b/tox.ini
@@ -57,6 +57,9 @@ deps =
 
 [testenv:integration-spark-thrift]
 description = run integration tests against a Spark thrift server
+allowlist_externals =
+    /bin/bash
+basepython = python3.8
 passenv =
     DBT_*
     PYTEST_ADDOPTS
@@ -64,10 +67,13 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.
-commands = pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*
+commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*'
 
 [testenv:integration-spark-session]
 description = run integration tests against a Spark session
+allowlist_externals =
+    /bin/bash
+basepython = python3.10
 passenv =
     DBT_*
     PYTEST_*
@@ -76,4 +82,4 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.[session]
-commands = pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*
+commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*'

From f0ef215e1c8186cf4270e695ec8663a5d745d127 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 15:08:48 +0200
Subject: [PATCH 08/15] Remove base pythons in tox.ini

---
 tox.ini | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 33055a211..31396b5ef 100644
--- a/tox.ini
+++ b/tox.ini
@@ -59,7 +59,6 @@ deps =
 description = run integration tests against a Spark thrift server
 allowlist_externals =
     /bin/bash
-basepython = python3.8
 passenv =
     DBT_*
     PYTEST_ADDOPTS
@@ -73,7 +72,6 @@ commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posarg
 description = run integration tests against a Spark session
 allowlist_externals =
     /bin/bash
-basepython = python3.10
 passenv =
     DBT_*
     PYTEST_*

From e8457df87d636324aae416c4a8eea363779f0156 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 15:19:19 +0200
Subject: [PATCH 09/15] Fix reference to Docker compose file

---
 .github/workflows/integration.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index d455e804b..517815e27 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -62,7 +62,7 @@ jobs:
 
       - uses: isbang/compose-action@v1.5.1
         with:
-          compose-file: "./docker/docker-compose.yml"
+          compose-file: "./docker-compose.yml"
 
       - name: Install tox
         run: |

From 842466a2883efd3a13826410f1477a0ff84c5e8f Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 15:42:11 +0200
Subject: [PATCH 10/15] Remove timeout

---
 .github/workflows/integration.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 517815e27..8eafa5c72 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -36,7 +36,6 @@ jobs:
     name: test with python ${{ matrix.python-version }}
 
     runs-on: ubuntu-latest
-    timeout-minutes: 10
 
     strategy:
       fail-fast: false

From 0738f2d0bcc5f30eab1cc92b4c82720ce99e3265 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 15:55:55 +0200
Subject: [PATCH 11/15] Remove artifact steps

---
 .github/workflows/integration.yml | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 8eafa5c72..9f26bd2be 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -73,14 +73,3 @@ jobs:
 
       - name: Run tox for Spark thrift
         run: tox -e integration-spark-thrift
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT  # Colons are not allowed in artifacts name
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: tests_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: tests_results.csv

From 277bef1a2a4368d54b2b1ce41b7894c51d4f7ef1 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 15:56:33 +0200
Subject: [PATCH 12/15] Bump Spark and Hadoop versions

---
 docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 85d01ba8a..a9b9e0a2c 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8
 FROM eclipse-temurin:${OPENJDK_VERSION}-jre
 
 ARG BUILD_DATE
-ARG SPARK_VERSION=3.1.3
-ARG HADOOP_VERSION=3.2
+ARG SPARK_VERSION=3.4.1
+ARG HADOOP_VERSION=3
 
 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \
       org.label-schema.build-date=$BUILD_DATE \

From 8d5853d3049c5e299ab7d824ab33fc374a9894ff Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 16:08:16 +0200
Subject: [PATCH 13/15] Reset Spark and Hadoop version

---
 docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a9b9e0a2c..85d01ba8a 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8
 FROM eclipse-temurin:${OPENJDK_VERSION}-jre
 
 ARG BUILD_DATE
-ARG SPARK_VERSION=3.4.1
-ARG HADOOP_VERSION=3
+ARG SPARK_VERSION=3.1.3
+ARG HADOOP_VERSION=3.2
 
 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \
       org.label-schema.build-date=$BUILD_DATE \

From 919528ab14dd731f9efa913d37b051bda8922e44 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 16:09:09 +0200
Subject: [PATCH 14/15] Update comment

---
 .github/workflows/integration.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 9f26bd2be..f4c34c5fb 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -2,7 +2,7 @@
 # Runs integration tests.
 
 # **why?**
-# Ensure code for dbt meets a certain quality standard.
+# Ensure code runs as expected.
 
 # **when?**
 # This will run for all PRs, when code is pushed to a release

From 15e48fd3f1f8d421f7f079a20ca8ba5fd5995d69 Mon Sep 17 00:00:00 2001
From: Cor Zuurmond <jczuurmond@protonmail.com>
Date: Fri, 29 Sep 2023 16:12:25 +0200
Subject: [PATCH 15/15] Add changie

---
 .changes/unreleased/Under the Hood-20230929-161218.yaml | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 .changes/unreleased/Under the Hood-20230929-161218.yaml

diff --git a/.changes/unreleased/Under the Hood-20230929-161218.yaml b/.changes/unreleased/Under the Hood-20230929-161218.yaml
new file mode 100644
index 000000000..c82e8252e
--- /dev/null
+++ b/.changes/unreleased/Under the Hood-20230929-161218.yaml	
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add Github action for integration testing
+time: 2023-09-29T16:12:18.968755+02:00
+custom:
+  Author: JCZuurmond
+  Issue: "719"