Merge branch 'main' into dependabot/github_actions/actions/checkout-4

.changes/unreleased/Fixes-20240605-202614.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Default to psycopg2-binary and allow overriding to psycopg2 via DBT_PSYCOPG2_NAME
+  (restores previous behavior)
+time: 2024-06-05T20:26:14.801254-04:00
+custom:
+  Author: mikealfare
+  Issue: "96"

.changes/unreleased/Fixes-20240626-163930.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix `persist_docs` for `materialized_view` materializations. Previously, using this configuration with materialized view models would lead to an error.
+time: 2024-06-26T16:39:30.455995+02:00
+custom:
+    Author: morsapaes
+    Issue: "120"

.changes/unreleased/Under the Hood-20240716-172442.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add support for experimental record/replay testing.
+time: 2024-07-16T17:24:42.271859-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "123"

.github/scripts/psycopg2-check.sh

@@ -0,0 +1,20 @@
+python -m venv venv
+source venv/bin/activate
+python -m pip install .
+
+if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then
+    if [[ $(pip show psycopg2-binary) ]]; then
+        PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2)
+        pip uninstall -y psycopg2-binary
+        pip install psycopg2==$PSYCOPG2_VERSION
+    fi
+fi
+
+PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2)
+if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then
+    echo -e 'Expected: "$PSYCOPG2_EXPECTED_NAME" but found: "$PSYCOPG2_NAME"'
+    exit 1
+fi
+
+deactivate
+rm -r ./venv

.github/workflows/docs-issue.yml

@@ -0,0 +1,41 @@
+# **what?**
+# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed
+
+# **why?**
+# To reduce barriers for keeping docs up to date
+
+# **when?**
+# When an issue is labeled `user docs` and is closed as completed.  Can be labeled before or after the issue is closed.
+
+
+name: Open issues in docs.getdbt.com repo when an issue is labeled
+run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}"
+
+on:
+  issues:
+    types: [labeled, closed]
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+    issues: write # comments on issues
+
+jobs:
+  open_issues:
+    # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned.
+    # If this logic does not exist in this workflow, it runs the
+    # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having
+    # generating the comment before the other runs.  This lives here instead of the shared workflow because this is where we
+    # decide if it should run or not.
+    if: |
+      (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && (
+      (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'user docs'))
+    uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
+    with:
+        issue_repository: "dbt-labs/docs.getdbt.com"
+        issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}"
+        issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated."
+    secrets: inherit

.github/workflows/integration-tests.yml

@@ -45,7 +45,7 @@ defaults:
 jobs:
   integration:
     name: Integration Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -91,7 +91,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run integration tests
-        run: hatch run integration-tests:all
+        run: hatch run integration-tests
         env:
           POSTGRES_TEST_HOST: localhost
           POSTGRES_TEST_PORT: 5432
@@ -102,24 +102,41 @@ jobs:
 
   psycopg2-check:
     name: "Test psycopg2 build version"
-    runs-on: ${{ matrix.scenario.platform }}
+    runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:
-        scenario:
-          - {platform: ubuntu-latest, psycopg2-name: psycopg2}
-          - {platform: macos-12, psycopg2-name: psycopg2-binary}
+        platform: [ubuntu-22.04, macos-12]
+        python-version: ["3.8", "3.11"]
     steps:
       - name: "Check out repository"
         uses: actions/checkout@v4
 
-      - name: "Test psycopg2 name"
-        run: |
-          python -m pip install .
-          PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary)
-          echo $PSYCOPG2_PIP_ENTRY
-          PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}"
-          echo $PSYCOPG2_NAME
-          if [[ "${PSYCOPG2_NAME}" != "${{ matrix.scenario.psycopg2-name }}" ]]; then
-            exit 1
-          fi
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: "Test psycopg2 name - default"
+        run: .github/scripts/psycopg2-check.sh
+        env:
+          PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+
+      - name: "Test psycopg2 name - invalid override"
+        run: .github/scripts/psycopg2-check.sh
+        env:
+          DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers
+          PSYCOPG2_EXPECTED_NAME: psycopg2-binary
+
+      - name: "Test psycopg2 name - override"
+        run: .github/scripts/psycopg2-check.sh
+        env:
+          DBT_PSYCOPG2_NAME: psycopg2
+          PSYCOPG2_EXPECTED_NAME: psycopg2-binary  # we have not implemented the hook yet, so this doesn't work
+
+      - name: "Test psycopg2 name - manual override"
+        # verify that the workaround documented in the `README.md` continues to work
+        run: .github/scripts/psycopg2-check.sh
+        env:
+          PSYCOPG2_WORKAROUND: true
+          PSYCOPG2_EXPECTED_NAME: psycopg2

.github/workflows/release-internal.yml

@@ -15,10 +15,6 @@ name: "Release internal patch"
 on:
   workflow_dispatch:
     inputs:
-      version_number:
-        description: "The release version number (i.e. 1.0.0b1)"
-        type: string
-        required: true
       ref:
         description: "The ref (sha or branch name) to use"
         type: string
@@ -29,6 +25,11 @@ on:
         type: string
         default: "python -c \"import dbt.adapters.postgres\""
         required: true
+      skip_tests:
+        description: "Should the tests be skipped? (default to false)"
+        type: boolean
+        required: true
+        default: false
 
 defaults:
   run:
@@ -41,9 +42,9 @@ jobs:
     uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main"
 
     with:
-      version_number: "${{ inputs.version_number }}"
       package_test_command: "${{ inputs.package_test_command }}"
       dbms_name: "postgres"
       ref: "${{ inputs.ref }}"
+      skip_tests: "${{ inputs.skip_tests }}"
 
     secrets: "inherit"

.github/workflows/release_prep_hatch.yml

@@ -31,7 +31,7 @@
 #  1. Bump the version if it has not been bumped
 #  2. Generate the changelog (via changie) if there is no markdown file for this version
 name: "Release prep"
-run-name: "Release prep: Generate changelog and bump ${{ inputs.package }} to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}"
+run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}"
 on:
   workflow_call:
     inputs:
@@ -342,7 +342,7 @@ jobs:
         uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
 
       - name: "Run unit tests"
-        run: hatch run unit-tests:all
+        run: hatch run unit-tests
 
   integration-tests:
     runs-on: ubuntu-latest
@@ -387,7 +387,7 @@ jobs:
         uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
 
       - name: "Run integration tests"
-        run: hatch run integration-tests:all
+        run: hatch run integration-tests
         env:
           POSTGRES_TEST_HOST: localhost
           POSTGRES_TEST_PORT: 5432
@@ -464,6 +464,6 @@ jobs:
         run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
       # if this is a real release and a release branch was created, delete it
-      - name: "Delete release branch: ${{ needs.branch.outputs.name }}"
+      - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}"
         if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }}
-        run: git push origin -d ${{ needs.branch.outputs.name }}
+        run: git push origin -d ${{ needs.release-branch.outputs.name }}

.github/workflows/unit-tests.yml

@@ -37,5 +37,5 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run unit tests
-        run: hatch run unit-tests:all
+        run: hatch run unit-tests
         shell: bash

.gitignore

@@ -161,3 +161,9 @@ cython_debug/
 
 # testing artifacts
 /logs
+
+# MacOS
+.DS_Store
+
+# vscode
+.vscode/

CONTRIBUTING.md

@@ -66,26 +66,42 @@ Rather than forking `dbt-labs/dbt-postgres`, use `dbt-labs/dbt-postgres` directl
 
 ### Installation
 
-1. Ensure the latest version of `pip` is installed:
+1. Ensure the latest versions of `pip` and `hatch` are installed:
    ```shell
-   pip install --upgrade pip
+   pip install --user --upgrade pip hatch
    ```
-2. Configure and activate a virtual environment using `virtualenv` as described in
-[Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment)
-3. Install `dbt-postgres` and development dependencies in the virtual environment
+2. This step is optional, but it's recommended. Configure `hatch` to create its virtual environments in the project. Add this block to your `hatch` `config.toml` file:
+   ```toml
+   # MacOS: ~/Library/Application Support/hatch/config.toml
+   [dirs.env]
+   virtual = ".hatch"
+   ```
+   This makes `hatch` create all virtual environments in the project root inside of the directory `/.hatch`, similar to `/.tox` for `tox`.
+   It also makes it easier to add this environment as a runner in common IDEs like VSCode and PyCharm.
+3. Create a `hatch` environment with all of the development dependencies and activate it:
+   ```shell
+   hatch run setup
+   hatch shell
+   ```
+4. Run any commands within the virtual environment by prefixing the command with `hatch run`:
    ```shell
-   pip install -e .[dev]
+   hatch run <command>
    ```
 
 When `dbt-postgres` is installed this way, any changes made to the `dbt-postgres` source code
 will be reflected in the virtual environment immediately.
 
-
 ## Testing
 
-`dbt-postgres` contains [unit](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit)
-and [functional](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional) tests.
+`dbt-postgres` contains [code quality checks](https://github.com/dbt-labs/dbt-postgres/tree/main/.pre-commit-config.yaml), [unit tests](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/unit),
+and [functional tests](https://github.com/dbt-labs/dbt-postgres/tree/main/tests/functional).
+
+### Code quality
 
+Code quality checks can run with a single command:
+```shell
+hatch run code-quality
+```
 
 ### Unit tests
 
@@ -94,10 +110,14 @@ Unit tests can be run locally without setting up a database connection:
 ```shell
 # Note: replace $strings with valid names
 
+# run all unit tests
+hatch run unit-test
+
 # run all unit tests in a module
-python -m pytest tests/unit/$test_file_name.py
+hatch run unit-tests tests/unit/$test_file_name.py
+
 # run a specific unit test
-python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name
+hatch run unit-tests tests/unit/$test_file_name.py::$test_class_name::$test_method_name
 ```
 
 ### Functional tests
@@ -120,16 +140,45 @@ Functional tests can be run locally with a valid database connection configured
 ```shell
 # Note: replace $strings with valid names
 
+# run all functional tests
+hatch run integration-tests
+
 # run all functional tests in a directory
-python -m pytest tests/functional/$test_directory
+hatch run integration-tests tests/functional/$test_directory
+
 # run all functional tests in a module
-python -m pytest tests/functional/$test_dir_and_filename.py
+hatch run integration-tests tests/functional/$test_directory/$test_filename.py
+
 # run all functional tests in a class
-python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name
+hatch run integration-tests tests/functional/$test_directory/$test_filename.py::$test_class_name
+
 # run a specific functional test
-python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name
+hatch run integration-tests tests/functional/$test_directory/$test_filename.py::$test_class_name::$test__method_name
+```
+
+### Testing against a development branch
+
+Some changes require a change in `dbt-common` and/or `dbt-adapters`.
+In that case, the dependency on `dbt-common` and/or `dbt-adapters` must be updated to point to the development branch. For example:
+
+```toml
+[tool.hatch.envs.default]
+dependencies = [
+    "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@my-dev-branch",
+    "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@my-dev-branch",
+    "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@my-dev-branch#subdirectory=dbt-tests-adapter",
+    ...,
+]
 ```
 
+This will install `dbt-common`/`dbt-adapters`/`dbt-tests-adapter` as snapshots. In other words, if `my-dev-branch` is updated on GitHub, those updates will not be reflected locally.
+In order to pick up those updates, the `hatch` environment(s) will need to be rebuilt:
+
+```shell
+exit
+hatch env prune
+hatch shell
+```
 
 ## Documentation
 

README.md

@@ -21,6 +21,20 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get
 - [Install dbt](https://docs.getdbt.com/docs/installation)
 - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
+### `psycopg2-binary` vs. `psycopg2`
+
+By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps:
+
+```bash
+if [[ $(pip show psycopg2-binary) ]]; then
+    PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2)
+    pip uninstall -y psycopg2-binary
+    pip install psycopg2==$PSYCOPG2_VERSION
+fi
+```
+
+This ensures the version of `psycopg2` will match that of `psycopg2-binary`.
+
 ## Join the dbt Community
 
 - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/)