Merge pull request #419 from basedosdados/feat/dbt-lint

feat: add dbt linter

.gitattributes

@@ -1 +1 @@
-*.sql linguist-detectable=true
+*.sql linguist-detectable=true

.github/workflows/cd.yaml

@@ -1,22 +1,18 @@
+---
 name: CD
-
 on:
   push:
-    branches:
-      - main
-
+    branches: [main]
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: basedosdados/queries-basedosdados-dev/dbt-rpc
-
 jobs:
   build-container:
     name: Build, publish and register
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -28,41 +24,35 @@ jobs:
             secret/data/gcp_credentials/basedosdados-dev    GCP_PROJECT_ID    | GCP_PROJECT_ID ;
             secret/data/gcp_credentials/basedosdados-dev    GKE_CLUSTER_NAME  | GKE_CLUSTER_NAME ;
             secret/data/gcp_credentials/basedosdados-dev    GKE_CLUSTER_ZONE  | GKE_CLUSTER_ZONE ;
-
       - name: Setup Google Cloud CLI
         uses: google-github-actions/[email protected]
         with:
           service_account_key: ${{ steps.import_secrets.outputs.GCP_SA_KEY_BASE64 }}
           project_id: ${{ steps.import_secrets.outputs.GKE_PROJECT_ID }}
           export_default_credentials: true
-
       - name: Get GKE credentials
         uses: google-github-actions/[email protected]
         with:
           cluster_name: ${{ steps.import_secrets.outputs.GKE_CLUSTER_NAME }}
           location: ${{ steps.import_secrets.outputs.GKE_CLUSTER_ZONE }}
           credentials: ${{ steps.import_secrets.outputs.GCP_SA_KEY_BASE64 }}
-
       - name: Log in to the Container registry
         uses: docker/[email protected]
         with:
           registry: ${{ env.REGISTRY}}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Build and push
         uses: docker/[email protected]
         with:
           context: .
           file: ./Dockerfile
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
-
       - name: Set up Kustomize
         run: |-
           curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64
           chmod u+x ./kustomize
-
       - name: Deploy
         run: |-
           ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE_DBT:TAG=$REGISTRY/$IMAGE_NAME:$GITHUB_SHA

.github/workflows/ci-dbt.yaml

@@ -0,0 +1,23 @@
+---
+name: CI dbt
+on:
+  pull_request:
+    paths: ['**.sql', '**.yaml']
+jobs:
+  lint:
+    name: Lint dbt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          cache: pip
+          python-version: '3.10'
+      - name: Install requirements
+        run: pip install -r requirements-dev.txt
+      - name: Lint sql
+        run: sqlfmt --diff .
+      - name: Lint yaml
+        run: yamlfix --exclude ".kubernetes/**/*" .

.github/workflows/lint_docker.yaml → .github/workflows/ci-docker.yaml

@@ -1,14 +1,15 @@
-name: Lint Dockerfile
-
+---
+name: CI Docker
 on:
   pull_request:
-
+    paths: [Dockerfile]
 jobs:
-  docker_lint:
-    name: Lint Dockerfile
+  lint:
+    name: Lint dockerfile
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Run hadolint
         uses: reviewdog/action-hadolint@v1
         with:

.github/workflows/docs.yaml

@@ -1,18 +1,15 @@
+---
 name: Deploy docs
-
 on:
   push:
-    branches:
-      - main
-
+    branches: [main]
 jobs:
   docs:
     name: Deploy docs
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -21,33 +18,26 @@ jobs:
           token: ${{ secrets.VAULT_TOKEN}}
           secrets: |
             secret/data/queries_credentials/<GCP_PROJECT_NAME>    GCP_SA_KEY | GCP_SA_KEY;
-
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.9.x"
-
+          python-version: 3.9.x
       - name: Upgrade pip
         run: |
           pip install -U pip
-
       - name: Install dependencies
         run: |
-          pip3 install -r requirements-actions.txt -r requirements.txt
-
+          pip3 install -r requirements-dev.txt -r requirements.txt
       - name: Setup credentials
         run: |
           echo ${{ steps.import_secrets.outputs.GCP_SA_KEY }} > dbt-sa.json
           python3 .github/workflows/scripts/modify_profiles.py
-
       - name: Generate docs
         run: |
           dbt docs generate --profiles-dir .
-
       - name: Delete credentials
         run: |
           rm dbt-sa.json
-
       - name: Deploy docs
         uses: JamesIves/[email protected]
         with:

.github/workflows/metadata_automation.yaml

@@ -1,18 +1,15 @@
+---
 name: Metadata Automation
-
 on:
   push:
-    branches:
-      - '*'
-
+    branches: ['*']
 jobs:
   update-metadata:
     name: Update metadata from Google Sheets
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
       - name: Import Secrets
         id: import_secrets
         uses: hashicorp/[email protected]
@@ -21,28 +18,23 @@ jobs:
           token: ${{ secrets.VAULT_TOKEN}}
           secrets: |
             secret/data/queries_credentials/basedosdados-dev    GCP_SA_KEY_BASE64 | GCP_SA_KEY_BASE64;
-
       - name: Setup Python version
         uses: actions/setup-python@v2
         with:
-          python-version: "3.9"
-
+          python-version: '3.9'
       - name: Install Python dependencies for actions
         run: |-
-          pip install -r requirements-actions.txt
-
+          pip install -r requirements-dev.txt
       - name: Checkout to the head ref
         run: |-
           git fetch --depth=1
           git checkout ${{ github.head_ref }}
-
       - name: Update metadata
         env:
           GCP_SA_KEY_BASE64: ${{ steps.import_secrets.outputs.GCP_SA_KEY_BASE64 }}
         run: |-
           python3 .github/workflows/scripts/metadata_automation.py
-
       - name: Commit changed files
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          commit_message: 'Update metadata'
+          commit_message: Update metadata

.gitignore

@@ -8,4 +8,4 @@ venv-dbt/
 .user.yml
 dbt_packages
 profiles.yml
-gists/test.py
+gists/test.py

.kubernetes/deployment.yaml

@@ -30,13 +30,12 @@ spec:
             - name: credentials-prod
               mountPath: /credentials-prod/
               readOnly: true
-          command:
-            - "/bin/sh"
-            - "-c"
+          command: [/bin/sh, -c]
           args:
             - >
               dbt deps;
-              dbt-rpc serve --profiles-dir . --profile default --target dev --host 0.0.0.0 --port 8580
+              dbt-rpc serve --profiles-dir . --profile default --target dev --host
+              0.0.0.0 --port 8580
       volumes:
         - name: credentials-dev
           secret:
@@ -45,7 +44,6 @@ spec:
           secret:
             secretName: credentials-prod
       restartPolicy: Always
-
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -78,13 +76,12 @@ spec:
             - name: credentials-prod
               mountPath: /credentials-prod/
               readOnly: true
-          command:
-            - "/bin/sh"
-            - "-c"
+          command: [/bin/sh, -c]
           args:
             - >
               dbt deps;
-              dbt-rpc serve --profiles-dir . --profile default --target prod --host 0.0.0.0 --port 8580
+              dbt-rpc serve --profiles-dir . --profile default --target prod --host
+              0.0.0.0 --port 8580
       volumes:
         - name: credentials-dev
           secret:

.kubernetes/service.yaml

@@ -7,12 +7,11 @@ metadata:
   name: dbt-rpc-dev
 spec:
   ports:
-    - name: "8580"
+    - name: '8580'
       port: 8580
       targetPort: 8580
   selector:
     app: dbt-rpc-dev
-
 ---
 apiVersion: v1
 kind: Service
@@ -22,7 +21,7 @@ metadata:
   name: dbt-rpc-prod
 spec:
   ports:
-    - name: "8580"
+    - name: '8580'
       port: 8580
       targetPort: 8580
   selector:

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+      - id: no-commit-to-branch
+        args: [-b, main]
+      - id: trailing-whitespace
+  - repo: https://github.com/tconbeer/sqlfmt
+    rev: v0.20.0
+    hooks:
+      - id: sqlfmt
+        language_version: python
+        additional_dependencies: ['.[jinjafmt]']
+  - repo: local
+    hooks:
+      - id: yamlfix
+        name: yamlfix
+        types: [yaml]
+        language: system
+        entry: yamlfix --exclude ".kubernetes/**/*" .

Makefile

@@ -21,3 +21,7 @@ logs:
 .PHONY: status
 status:
 	docker-compose ps
+
+.PHONY: lint
+lint:
+	pre-commit run --all-files