diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 8063fbfc..03508e7f 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Import Secrets id: import_secrets uses: hashicorp/vault-action@v2.4.1 diff --git a/.github/workflows/ci-dbt.yaml b/.github/workflows/ci-dbt.yaml index f687ebd7..cda880c9 100644 --- a/.github/workflows/ci-dbt.yaml +++ b/.github/workflows/ci-dbt.yaml @@ -10,14 +10,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up poetry + run: pipx install poetry - name: Set up python uses: actions/setup-python@v4 with: - cache: pip + cache: poetry python-version: '3.10' - name: Install requirements - run: pip install -r requirements-dev.txt + run: poetry install --only=dev - name: Lint sql - run: sqlfmt --diff . + run: poetry run sqlfmt --diff . - name: Lint yaml - run: yamlfix --exclude ".kubernetes/**/*" . + run: poetry run yamlfix --exclude ".kubernetes/**/*" . diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a37d0c27..b8e6fd98 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Import Secrets id: import_secrets uses: hashicorp/vault-action@v2.4.1 @@ -18,23 +18,22 @@ jobs: token: ${{ secrets.VAULT_TOKEN}} secrets: | secret/data/queries_credentials/ GCP_SA_KEY | GCP_SA_KEY; + - name: Set up poetry + run: pipx install poetry - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.9.x - - name: Upgrade pip - run: | - pip install -U pip - - name: Install dependencies - run: | - pip3 install -r requirements-dev.txt -r requirements.txt + cache: poetry + python-version: '3.10' + - name: Install requirements + run: poetry install --with=dev - name: Setup credentials run: | echo ${{ steps.import_secrets.outputs.GCP_SA_KEY }} > dbt-sa.json - python3 .github/workflows/scripts/modify_profiles.py + poetry run python .github/workflows/scripts/modify_profiles.py - name: Generate docs run: | - dbt docs generate --profiles-dir . + poetry run dbt docs generate --profiles-dir . - name: Delete credentials run: | rm dbt-sa.json diff --git a/.github/workflows/metadata_automation.yaml b/.github/workflows/metadata_automation.yaml index 842696c2..641b9f29 100644 --- a/.github/workflows/metadata_automation.yaml +++ b/.github/workflows/metadata_automation.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Import Secrets id: import_secrets uses: hashicorp/vault-action@v2.4.1 @@ -18,13 +18,13 @@ jobs: token: ${{ secrets.VAULT_TOKEN}} secrets: | secret/data/queries_credentials/basedosdados-dev GCP_SA_KEY_BASE64 | GCP_SA_KEY_BASE64; - - name: Setup Python version - uses: actions/setup-python@v2 + - name: Set up poetry + run: pipx install poetry + - name: Set up Python + uses: actions/setup-python@v4 with: - python-version: '3.9' - - name: Install Python dependencies for actions - run: |- - pip install -r requirements-dev.txt + cache: poetry + python-version: '3.10' - name: Checkout to the head ref run: |- git fetch --depth=1 @@ -33,7 +33,7 @@ jobs: env: GCP_SA_KEY_BASE64: ${{ steps.import_secrets.outputs.GCP_SA_KEY_BASE64 }} run: |- - python3 .github/workflows/scripts/metadata_automation.py + poetry run python .github/workflows/scripts/metadata_automation.py - name: Commit changed files uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/Dockerfile b/Dockerfile index e1f199e8..f24e35d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,30 @@ -FROM python:3.8-slim +# Build Image -# Setup virtual environment -ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" +FROM python:3.10-bookworm AS builder -# Copy and install dependencies -WORKDIR /tmp -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt && rm requirements.txt +RUN pip install poetry==1.7.0 + +ENV POETRY_VIRTUALENVS_CREATE=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_CACHE_DIR=/tmp/pypoetry + +WORKDIR /app +COPY pyproject.toml poetry.lock dbt_project.yml packages.yml ./ +RUN poetry install --no-root && poetry run dbt deps && rm -rf $POETRY_CACHE_DIR + +# Runtime Image + +FROM python:3.10-slim-bookworm AS runner + +ENV VIRTUAL_ENV=/app/.venv \ + DBT_PACKAGES=/app/dbt_packages \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} +COPY --from=builder ${DBT_PACKAGES} ${DBT_PACKAGES} -# Copy dbt project and profiles -WORKDIR /dbt COPY . . -# Run dbt deps and dbt rpc -CMD ["/dbt/start-server.sh"] +CMD ["dbt-rpc", "serve", "--profiles-dir", ".", "--host", "0.0.0.0", "--port", "8580"] diff --git a/start-server.sh b/start-server.sh deleted file mode 100755 index ea2a6a74..00000000 --- a/start-server.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env sh - -dbt deps -dbt-rpc serve --profiles-dir "." --host "0.0.0.0" --port "8580"