Merge pull request #422 from basedosdados/feat/unify-reqs

feat: unify requirements in pyproject

.github/workflows/cd.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]

.github/workflows/ci-dbt.yaml

@@ -10,14 +10,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Set up poetry
+        run: pipx install poetry
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          cache: pip
+          cache: poetry
           python-version: '3.10'
       - name: Install requirements
-        run: pip install -r requirements-dev.txt
+        run: poetry install --only=dev
       - name: Lint sql
-        run: sqlfmt --diff .
+        run: poetry run sqlfmt --diff .
       - name: Lint yaml
-        run: yamlfix --exclude ".kubernetes/**/*" .
+        run: poetry run yamlfix --exclude ".kubernetes/**/*" .

.github/workflows/docs.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -18,23 +18,22 @@ jobs:
           token: ${{ secrets.VAULT_TOKEN}}
           secrets: |
             secret/data/queries_credentials/<GCP_PROJECT_NAME>    GCP_SA_KEY | GCP_SA_KEY;
+      - name: Set up poetry
+        run: pipx install poetry
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9.x
-      - name: Upgrade pip
-        run: |
-          pip install -U pip
-      - name: Install dependencies
-        run: |
-          pip3 install -r requirements-dev.txt -r requirements.txt
+          cache: poetry
+          python-version: '3.10'
+      - name: Install requirements
+        run: poetry install --with=dev
       - name: Setup credentials
         run: |
           echo ${{ steps.import_secrets.outputs.GCP_SA_KEY }} > dbt-sa.json
-          python3 .github/workflows/scripts/modify_profiles.py
+          poetry run python .github/workflows/scripts/modify_profiles.py
       - name: Generate docs
         run: |
-          dbt docs generate --profiles-dir .
+          poetry run dbt docs generate --profiles-dir .
       - name: Delete credentials
         run: |
           rm dbt-sa.json

.github/workflows/metadata_automation.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -18,13 +18,13 @@ jobs:
           token: ${{ secrets.VAULT_TOKEN}}
           secrets: |
             secret/data/queries_credentials/basedosdados-dev    GCP_SA_KEY_BASE64 | GCP_SA_KEY_BASE64;
-      - name: Setup Python version
-        uses: actions/setup-python@v2
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
-      - name: Install Python dependencies for actions
-        run: |-
-          pip install -r requirements-dev.txt
+          cache: poetry
+          python-version: '3.10'
       - name: Checkout to the head ref
         run: |-
           git fetch --depth=1
@@ -33,7 +33,7 @@ jobs:
         env:
           GCP_SA_KEY_BASE64: ${{ steps.import_secrets.outputs.GCP_SA_KEY_BASE64 }}
         run: |-
-          python3 .github/workflows/scripts/metadata_automation.py
+          poetry run python .github/workflows/scripts/metadata_automation.py
       - name: Commit changed files
         uses: stefanzweifel/git-auto-commit-action@v4
         with:

Dockerfile

@@ -1,18 +1,30 @@
-FROM python:3.8-slim
+# Builder Image
 
-# Setup virtual environment
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+FROM python:3.10-bookworm AS builder
 
-# Copy and install dependencies
-WORKDIR /tmp
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && rm requirements.txt
+RUN pip install --no-cache-dir poetry==1.7.0
+
+ENV POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_CACHE_DIR=/tmp/pypoetry
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock dbt_project.yml packages.yml ./
+RUN poetry install --no-root && poetry run dbt deps && rm -rf $POETRY_CACHE_DIR
+
+# Runner Image
+
+FROM python:3.10-slim-bookworm AS runner
+
+ENV VIRTUAL_ENV=/app/.venv \
+    DBT_PACKAGES=/app/dbt_packages \
+    PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY --from=builder ${DBT_PACKAGES} ${DBT_PACKAGES}
 
-# Copy dbt project and profiles
-WORKDIR /dbt
 COPY . .
 
-# Run dbt deps and dbt rpc
-CMD ["/dbt/start-server.sh"]
+CMD ["dbt-rpc", "serve", "--profiles-dir", ".", "--host", "0.0.0.0", "--port", "8580"]