feat: improve ci in github actions (#562)

basedosdados · Feb 10, 2024 · 2904b56 · 2904b56
1 parent d4fb621
commit 2904b56
Show file tree

Hide file tree

Showing 69 changed files with 864 additions and 922 deletions.
diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml
@@ -1,12 +1,10 @@
+---
 name: Deployment (Development)
-
 on:
   workflow_run:
-    workflows: ["Release Image (Development)"]
-    types:
-      - completed
+    workflows: [Release Image (Development)]
+    types: [completed]
   workflow_dispatch:
-
 jobs:
   deploy-dev:
     runs-on: ubuntu-latest
@@ -16,7 +14,6 @@ jobs:
     environment:
       name: development
       url: https://api.development.basedosdados.org
-
     steps:
       - name: Download branch name
         uses: actions/github-script@v6
@@ -38,19 +35,15 @@ jobs:
             });
             let fs = require('fs');
             fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/artifact.zip`, Buffer.from(download.data));
-
       - name: Extract branch name
         run: unzip artifact.zip
-
       - name: Read branch name
         id: extract_branch
         run: echo "branch=$(cat branch)" >> "$GITHUB_OUTPUT"
-
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ steps.extract_branch.outputs.branch }}
-
       - name: Import secrets from Vault
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -62,21 +55,18 @@ jobs:
             secret/data/gcp_credentials/basedosdados-dev GH_ACTIONS_SA    | GCP_SA ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
-
       - name: Setup Google Cloud CLI
         uses: google-github-actions/[email protected]
         with:
           service_account_key: ${{ steps.import_secrets.outputs.GCP_SA }}
           project_id: ${{ steps.import_secrets.outputs.GCP_PROJECT_ID }}
           export_default_credentials: true
-
       - name: Get GKE credentials
         uses: google-github-actions/[email protected]
         with:
           cluster_name: ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}
           location: ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
           credentials: ${{ steps.import_secrets.outputs.GCP_SA }}
-
       - name: Write values.yaml file
         run: |
           cat << EOF > values.yaml
@@ -147,9 +137,8 @@ jobs:
                   - development.api.basedosdados.org
                 secretName: api-development-basedosdados-org-tls
           EOF
-
       - name: Deploy using Helm
-        run: |
+        run: |-
           helm upgrade \
             --wait \
             --install \

diff --git a/.github/workflows/cd-prod.yaml b/.github/workflows/cd-prod.yaml
@@ -1,14 +1,11 @@
+---
 name: Deployment (Production)
-
 on:
   workflow_run:
-    workflows: ["Release Image (Production)"]
-    branches:
-      - main
-    types:
-      - completed
+    workflows: [Release Image (Production)]
+    branches: [main]
+    types: [completed]
   workflow_dispatch:
-
 jobs:
   deploy-prod:
     runs-on: ubuntu-latest
@@ -18,13 +15,11 @@ jobs:
     environment:
       name: production
       url: https://api.basedosdados.org
-
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
-
       - name: Import secrets from Vault
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -36,21 +31,18 @@ jobs:
             secret/data/gcp_credentials/basedosdados-dev GH_ACTIONS_SA    | GCP_SA ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
-
       - name: Setup Google Cloud CLI
         uses: google-github-actions/[email protected]
         with:
           service_account_key: ${{ steps.import_secrets.outputs.GCP_SA }}
           project_id: ${{ steps.import_secrets.outputs.GCP_PROJECT_ID }}
           export_default_credentials: true
-
       - name: Get GKE credentials
         uses: google-github-actions/[email protected]
         with:
           cluster_name: ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}
           location: ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
           credentials: ${{ steps.import_secrets.outputs.GCP_SA }}
-
       - name: Write values.yaml file
         run: |
           cat << EOF > values.yaml
@@ -121,9 +113,8 @@ jobs:
                   - api.basedosdados.org
                 secretName: api-basedosdados-org-tls
           EOF
-
       - name: Deploy using Helm
-        run: |
+        run: |-
           helm upgrade \
             --wait \
             --install \

diff --git a/.github/workflows/cd-staging.yaml b/.github/workflows/cd-staging.yaml
@@ -1,14 +1,11 @@
+---
 name: Deployment (Staging)
-
 on:
   workflow_run:
-    workflows: ["Release Image (Staging)"]
-    branches:
-      - main
-    types:
-      - completed
+    workflows: [Release Image (Staging)]
+    branches: [main]
+    types: [completed]
   workflow_dispatch:
-
 jobs:
   deploy-staging:
     runs-on: ubuntu-latest
@@ -18,13 +15,11 @@ jobs:
     environment:
       name: staging
       url: https://staging.api.basedosdados.org
-
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
-
       - name: Import secrets from Vault
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -36,21 +31,18 @@ jobs:
             secret/data/gcp_credentials/basedosdados-dev GH_ACTIONS_SA    | GCP_SA ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_NAME | GKE_CLUSTER_NAME ;
             secret/data/gcp_credentials/basedosdados-dev GKE_CLUSTER_ZONE | GKE_CLUSTER_ZONE ;
-
       - name: Setup Google Cloud CLI
         uses: google-github-actions/[email protected]
         with:
           service_account_key: ${{ steps.import_secrets.outputs.GCP_SA }}
           project_id: ${{ steps.import_secrets.outputs.GCP_PROJECT_ID }}
           export_default_credentials: true
-
       - name: Get GKE credentials
         uses: google-github-actions/[email protected]
         with:
           cluster_name: ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}
           location: ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
           credentials: ${{ steps.import_secrets.outputs.GCP_SA }}
-
       - name: Write values.yaml file
         run: |
           cat << EOF > values.yaml
@@ -121,9 +113,8 @@ jobs:
                   - staging.api.basedosdados.org
                 secretName: staging-api-basedosdados-org-tls
           EOF
-
       - name: Deploy using Helm
-        run: |
+        run: |-
           helm upgrade \
             --wait \
             --install \

diff --git a/.github/workflows/ci-dbt.yaml b/.github/workflows/ci-dbt.yaml
@@ -0,0 +1,25 @@
+---
+name: CI dbt
+on:
+  pull_request:
+    paths: ['**.sql', '**.yaml']
+jobs:
+  lint:
+    name: Lint dbt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          cache: poetry
+          python-version: '3.10'
+      - name: Install requirements
+        run: poetry install --only=dev
+      - name: Lint sql
+        run: poetry run sqlfmt --diff .
+      - name: Lint yaml
+        run: poetry run yamlfix --exclude "charts/**/*" .
diff --git a/.github/workflows/ci-docker.yaml b/.github/workflows/ci-docker.yaml
@@ -0,0 +1,16 @@
+---
+name: CI Docker
+on:
+  pull_request:
+    paths: [Dockerfile]
+jobs:
+  lint:
+    name: Lint dockerfile
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run hadolint
+        uses: reviewdog/action-hadolint@v1
+        with:
+          reporter: github-pr-review
diff --git a/.github/workflows/ci-pr.yaml b/.github/workflows/ci-pr.yaml
@@ -0,0 +1,19 @@
+---
+name: CI Pull Request
+on:
+  pull_request:
+jobs:
+  lint:
+    name: Lint pull request
+    runs-on: ubuntu-latest
+    steps:
+      - name: Lint pull request title
+        uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          subjectPattern: ^(?![A-Z]).+$
+          subjectPatternError: |-
+            The subject "{subject}" found in the pull request title "{title}"
+            didn't match the configured pattern. Please ensure that the subject
+            doesn't start with an uppercase character.
diff --git a/.github/workflows/ci-python.yaml b/.github/workflows/ci-python.yaml
@@ -0,0 +1,42 @@
+---
+name: CI Python
+on:
+  pull_request:
+    paths: ['**.py']
+jobs:
+  lint:
+    name: Lint python
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Lint source code
+        uses: chartboost/ruff-action@v1
+  test:
+    name: Test python
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up poetry
+        run: pipx install poetry
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          cache: poetry
+          architecture: x64
+          python-version: '3.11'
+      - name: Install requirements
+        run: poetry install --only=test
+      - name: Test
+        run: |
+          poetry run pytest \
+            --junitxml=pytest.xml \
+            --cov-report=term-missing:skip-covered \
+            --cov=bd_api bd_api | tee pytest-coverage.txt
+      - name: Report coverage
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          hide-report: true
+          junitxml-path: ./pytest.xml
+          pytest-coverage-path: ./pytest-coverage.txt
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml