Merge pull request #440 from basedosdados/feat/unify-reqs

feat: unify requirements in pyproject
basedosdados · Feb 16, 2024 · 8a4c4d7 · 8a4c4d7
2 parents 4bb19ed + 938d552
commit 8a4c4d7
Show file tree

Hide file tree

Showing 12 changed files with 2,307 additions and 79 deletions.
diff --git a/.github/workflows/deploy.yaml → .github/workflows/cd.yaml b/.github/workflows/deploy.yaml → .github/workflows/cd.yaml
@@ -1,5 +1,5 @@
 ---
-name: Deployment
+name: CD
 on:
   pull_request:
     types: [closed]
@@ -15,13 +15,14 @@ env:
   IMAGE_NAME: ghcr.io/basedosdados/queries-basedosdados
 jobs:
   build-container:
-    if: (github.event.pull_request.merged == true && github.event.pull_request.base.ref
-      == 'main')
+    if: |
+      github.event.pull_request.merged == true
+      && github.event.pull_request.base.ref == 'main'
     name: Deployment
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Setup Google Cloud CLI
         uses: google-github-actions/[email protected]
         with:
@@ -62,37 +63,39 @@ jobs:
         uses: tj-actions/changed-files@v35
         with:
           separator: ','
-      - name: Setup Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          cache: poetry
+          python-version: '3.9'
+      - name: Install requirements
+        run: poetry install --only=dev
       - name: Run script for approving table
         run: |
-          python .github/workflows/scripts/table_approve.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --source-bucket-name ${{ secrets.SOURCE_BUCKET_NAME }} --destination-bucket-name ${{ secrets.DESTINATION_BUCKET_NAME }} --backup-bucket-name ${{ secrets.BACKUP_BUCKET_NAME }} --prefect-backend-token ${{ secrets.PREFECT_BACKEND_TOKEN }} --materialization-mode ${{ secrets.MATERIALIZATION_MODE }} --materialization-label ${{ secrets.MATERIALIZATION_LABEL }}
+          poetry run python .github/workflows/scripts/table_approve.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --source-bucket-name ${{ secrets.SOURCE_BUCKET_NAME }} --destination-bucket-name ${{ secrets.DESTINATION_BUCKET_NAME }} --backup-bucket-name ${{ secrets.BACKUP_BUCKET_NAME }} --prefect-backend-token ${{ secrets.PREFECT_BACKEND_TOKEN }} --materialization-mode ${{ secrets.MATERIALIZATION_MODE }} --materialization-label ${{ secrets.MATERIALIZATION_LABEL }}
   change-metadata-status:
     needs: table-approve
     name: Change metadata status to "production"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get all changed files using a comma separator
         id: changed-files
         uses: tj-actions/changed-files@v35
         with:
           separator: ','
-      - name: Setup Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          cache: poetry
+          python-version: '3.9'
+      - name: Install requirements
+        run: poetry install --only=dev
       - name: Run script for changing metadata status
         run: |-
           python .github/workflows/scripts/change_metadata_status.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }} --status published --email ${{ secrets.BACKEND_EMAIL }} --password ${{ secrets.BACKEND_PASSWORD }}
diff --git a/.github/workflows/ci-dbt.yaml b/.github/workflows/ci-dbt.yaml
@@ -10,14 +10,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Set up poetry
+        run: pipx install poetry
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          cache: pip
-          python-version: '3.10'
+          cache: poetry
+          python-version: '3.9'
       - name: Install requirements
-        run: pip install -r requirements-dev.txt
+        run: poetry install --only=dev
       - name: Lint sql
-        run: sqlfmt --diff .
+        run: poetry run sqlfmt --diff .
       - name: Lint yaml
-        run: yamlfix --exclude ".kubernetes/**/*" .
+        run: poetry run yamlfix --exclude ".kubernetes/**/*" .
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -7,30 +7,30 @@ env:
   DBT_SA: ${{ secrets.DBT_SA }}
 jobs:
   docs:
-    if: github.event.pull_request.merged == true && github.event.pull_request.base.ref
-      == 'main'
+    if: |
+      github.event.pull_request.merged == true
+      && github.event.pull_request.base.ref == 'main'
     name: Deploy docs
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v4
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9.x
-      - name: Upgrade pip
-        run: |
-          pip install -U pip
-      - name: Install dependencies
-        run: |
-          pip3 install -r requirements-docs.txt -r requirements.txt
+          cache: poetry
+          python-version: '3.9'
+      - name: Install requirements
+        run: poetry install --with=dev
       - name: Setup credentials
         run: |
           echo $DBT_SA | base64 --decode > dbt-sa.json
-          python3 .github/workflows/scripts/modify_profiles.py
+          poetry run python .github/workflows/scripts/modify_profiles.py
       - name: Generate docs
         run: |
-          dbt docs generate --profiles-dir .
+          poetry run dbt docs generate --profiles-dir .
       - name: Delete credentials
         run: |
           rm dbt-sa.json

diff --git a/.github/workflows/sync-dbt-schema.yaml b/.github/workflows/sync-dbt-schema.yaml
@@ -12,25 +12,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
       - name: Get all changed files using a comma separator
         id: changed-files
         uses: tj-actions/changed-files@v35
         with:
           separator: ','
-      - name: Setup Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          cache: poetry
+          python-version: '3.9'
+      - name: Install requirements
+        run: poetry install --only=dev
       - name: Run script for syncing DBT schema
         run: |
-          python .github/workflows/scripts/sync_dbt_schema.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }}
+          poetry run python .github/workflows/scripts/sync_dbt_schema.py --modified-files ${{ steps.changed-files.outputs.all_modified_files }} --graphql-url ${{ secrets.BACKEND_GRAPHQL_URL }}
       - name: Commit changed files
         uses: stefanzweifel/git-auto-commit-action@v4
         with:

diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,6 @@ dbt_packages/
 # Python
 **/test.py
 __pycache__/
-pyproject.toml
 #
 models/*/tmp/
 models/*/input/

diff --git a/Dockerfile b/Dockerfile
@@ -1,18 +1,30 @@
-FROM python:3.8-slim
+# Builder Image
 
-# Setup virtual environment
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+FROM python:3.9-bookworm AS builder
 
-# Copy and install dependencies
-WORKDIR /tmp
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && rm requirements.txt
+RUN pip install --no-cache-dir poetry==1.7.0
+
+ENV POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_CACHE_DIR=/tmp/pypoetry
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock dbt_project.yml packages.yml ./
+RUN poetry install --no-root && poetry run dbt deps && rm -rf $POETRY_CACHE_DIR
+
+# Runner Image
+
+FROM python:3.9-slim-bookworm AS runner
+
+ENV VIRTUAL_ENV=/app/.venv \
+    DBT_PACKAGES=/app/dbt_packages \
+    PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY --from=builder ${DBT_PACKAGES} ${DBT_PACKAGES}
 
-# Copy dbt project and profiles
-WORKDIR /dbt
 COPY . .
 
-# Run dbt deps and dbt rpc
-CMD ["/dbt/start-server.sh"]
+CMD ["dbt-rpc", "serve", "--profiles-dir", ".", "--host", "0.0.0.0", "--port", "8580"]