From 938fc07fcced09614e2c7026c2f3a262c4127b30 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 23:30:50 -0400 Subject: [PATCH] update to align with other docker release pipelines --- .github/workflows/release.yml | 14 ++--- dbt/adapters/postgres/relation.py | 30 ++++++----- docker/Dockerfile | 34 ++++++------ docker/README.md | 20 ++----- docker/dev.Dockerfile | 54 +++++++++++++++++++ docker/test.sh | 2 +- pyproject.toml | 6 +++ .../shared_tests/test_hooks/test_hooks.py | 1 + .../test_simple_seed/test_simple_seed.py | 1 + tests/unit/test_renamed_relations.py | 12 +++-- 10 files changed, 114 insertions(+), 60 deletions(-) create mode 100644 docker/dev.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a10eed6c..89f4baa4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,6 @@ jobs: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing outputs: version: ${{ steps.release-inputs.outputs.version }} - is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} steps: - name: Check out repository @@ -48,12 +47,6 @@ jobs: echo "archive-name=$archive_name" >> $GITHUB_OUTPUT echo "version=version" >> $GITHUB_OUTPUT - - name: Audit version to determine if it is a pre-release - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.release-inputs.outputs.version }} - - name: Build `dbt-postgres` uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main with: @@ -68,10 +61,11 @@ jobs: docker-release: name: "Docker Release" needs: [release] - if: ${{ needs.release.outputs.is-docker-release }} + if: ${{ !failure() && !cancelled() }} permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: "dbt-postgres" + package: ${{ github.event.repository.name }} version_number: ${{ needs.release.outputs.version }} + test_run: ${{ inputs.deploy-to == 'test' }} diff --git a/dbt/adapters/postgres/relation.py b/dbt/adapters/postgres/relation.py index 677b12ac..05d55237 100644 --- a/dbt/adapters/postgres/relation.py +++ b/dbt/adapters/postgres/relation.py @@ -20,19 +20,23 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - renameable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - } - )) - replaceable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - } - )) + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + ) + ) + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + ) def __post_init__(self): # Check for length of Postgres table/view names. diff --git a/docker/Dockerfile b/docker/Dockerfile index a87beac3..b29058be 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,18 @@ -ARG build_for=linux/amd64 +# this image gets published to GHCR for production use +ARG py_version=3.10.7 -FROM --platform=$build_for python:3.10.7-slim-bullseye as base - -# ref is updated automatically every final release via bumpversion -ARG dbt_postgres_ref=dbt-postgres@v1.7.10 +FROM python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ - git \ - ssh-client \ - software-properties-common \ - make \ - build-essential \ - ca-certificates \ - libpq-dev \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -24,10 +22,16 @@ RUN apt-get update \ ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-postgres + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -FROM base as dbt-postgres -RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/${dbt_postgres_ref}" +RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres@${commit_ref}" diff --git a/docker/README.md b/docker/README.md index 21deea2c..22af3fe9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,11 +14,12 @@ docker build --tag --target dbt-postgres --- -By default the images will be populated with the most recent release of `dbt-postgres`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +By default the image will be populated with the latest version of `dbt-postgres` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: ```shell docker build --tag \ --target dbt-postgres \ - --build-arg dbt_postgres_ref= \ + --build-arg commit_ref= \ ``` @@ -35,23 +36,10 @@ cd dbt-core/docker docker build \ --tag my-other-dbt \ --target dbt-postgres \ - --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ . ``` -## Special cases -There are a few special cases worth noting: - -* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -```shell -docker build --tag my_dbt \ - --target dbt-postgres \ - --build-arg build_for=linux/arm64 \ - -``` - -Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). - ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ```shell diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 00000000..a7d2eca3 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,54 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + libpq-dev=14.11-0ubuntu0.22.04.1 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + +# install python dependencies +RUN python3 -m pip install --upgrade --no-cache-dir "hatch==1.9.1" + + +FROM base as dbt-postgres-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv diff --git a/docker/test.sh b/docker/test.sh index f7b86352..32d25362 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -17,6 +17,6 @@ clear \ "#########################################\n"\ && docker build --tag dbt-postgres-1.0.0b1 \ --target dbt-postgres \ - --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ docker \ && docker run dbt-postgres-1.0.0b1 --version diff --git a/pyproject.toml b/pyproject.toml index 996ea46a..0b212bcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,12 @@ dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", ] +[tool.hatch.envs.default.scripts] +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-postgres-dev .", + "docker run --rm -it --name dbt-postgres-dev -v $(pwd):/opt/code dbt-postgres-dev", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-postgres ." [tool.hatch.envs.lint] detached = true diff --git a/tests/functional/shared_tests/test_hooks/test_hooks.py b/tests/functional/shared_tests/test_hooks/test_hooks.py index 84381130..7e832038 100644 --- a/tests/functional/shared_tests/test_hooks/test_hooks.py +++ b/tests/functional/shared_tests/test_hooks/test_hooks.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it uses a `data` directory. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.hooks.test_model_hooks import ( BasePrePostModelHooks, BaseHookRefs, diff --git a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py index cd849788..61664ca9 100644 --- a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py +++ b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it creates a `data` directory at run time. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.simple_seed.test_seed import ( BaseBasicSeedTests, BaseSeedConfigFullRefreshOn, diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py index 49900d8e..29bbabf2 100644 --- a/tests/unit/test_renamed_relations.py +++ b/tests/unit/test_renamed_relations.py @@ -9,8 +9,10 @@ def test_renameable_relation(): identifier="my_table", type=RelationType.Table, ) - assert relation.renameable_relations == frozenset({ - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - }) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + )