From c024325fd174cfc627a53d27dd674589495e978d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Mon, 1 Apr 2024 16:51:43 -0300 Subject: [PATCH 1/6] feat: add elementary deploy action --- .github/workflows/elementary.yml | 38 ++++++++++++++++++++++++++++++++ .gitignore | 1 + packages.yml | 2 +- poetry.lock | 13 ++++++----- profiles.yml | 9 ++++++++ pyproject.toml | 2 +- 6 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/elementary.yml diff --git a/.github/workflows/elementary.yml b/.github/workflows/elementary.yml new file mode 100644 index 00000000..51242931 --- /dev/null +++ b/.github/workflows/elementary.yml @@ -0,0 +1,38 @@ +name: Deploy Elementary Report +on: + push: + branches: [ "main", "master" ] + workflow_dispatch: +jobs: + elementary: + runs-on: ubuntu-latest + steps: + - name: Checkout dbt project + uses: actions/checkout@v3 + - name: Run Elementary + uses: elementary-data/run-elementary-action@v1.12 + with: + warehouse-type: bigquery + adapter-version: 1.5.9 + profiles-yml: ${{ secrets.ELEMENTARY_PROFILES_YML }} + edr-command: + edr report + --file-path "report.html" + && + edr send-report + --google-service-account-path "/tmp/gcs_keyfile.json" + --gcs-bucket-name "${{ secrets.GCS_BUCKET_NAME }}" + --update-bucket-website "true" + bigquery-keyfile: ${{ secrets.BIGQUERY_KEYFILE }} + gcs-keyfile: ${{ secrets.GCS_KEYFILE }} + - name: Upload report + uses: actions/upload-artifact@v3 + with: + name: report.html + path: report.html + - name: Upload log + if: always() + uses: actions/upload-artifact@v3 + with: + name: edr.log + path: edr.log diff --git a/.gitignore b/.gitignore index 0c82440d..40d9da05 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ models/*/input/ models/*/output/ models/*/extra/ venv* +edr_target/ \ No newline at end of file diff --git a/packages.yml b/packages.yml index ed494411..72fc963f 100644 --- a/packages.yml +++ b/packages.yml @@ -3,4 +3,4 @@ packages: - package: dbt-labs/dbt_utils version: 1.1.1 - package: elementary-data/elementary - version: 0.14.0 + version: 0.14.1 diff --git a/poetry.lock b/poetry.lock index 604a92a1..e7367342 100644 --- a/poetry.lock +++ b/poetry.lock @@ -372,19 +372,19 @@ pyarrow = ">=3.0.0" [[package]] name = "dbt-bigquery" -version = "1.5.6" +version = "1.5.9" description = "The Bigquery adapter plugin for dbt" optional = false python-versions = ">=3.8" files = [ - {file = "dbt-bigquery-1.5.6.tar.gz", hash = "sha256:4655cf2ee0acda986b80e6c5d55cae57871bef22d868dfe29d8d4a5bca98a1ba"}, - {file = "dbt_bigquery-1.5.6-py3-none-any.whl", hash = "sha256:3f37544716880cbd17b32bc0c9728a0407b5615b2cd08e1bb904a7a83c46eb6c"}, + {file = "dbt-bigquery-1.5.9.tar.gz", hash = "sha256:06e3f8ebb8f696f028084fb9fc1c934b472b28996a76a051f65c548712e15651"}, + {file = "dbt_bigquery-1.5.9-py3-none-any.whl", hash = "sha256:b63ce45ea34dab8fcfa0592f9843cb35d6744ddd1a84efff577971964fa85db2"}, ] [package.dependencies] agate = ">=1.6.3,<1.7.0" dbt-core = ">=1.5.0,<1.6.0" -google-cloud-bigquery = ">=3.0,<4.0" +google-cloud-bigquery = {version = ">=3.0,<4.0", extras = ["pandas"]} google-cloud-dataproc = ">=5.0,<6.0" google-cloud-storage = ">=2.4,<3.0" @@ -646,10 +646,13 @@ files = [ ] [package.dependencies] +db-dtypes = {version = ">=0.3.0,<2.0.0dev", optional = true, markers = "extra == \"pandas\""} google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" packaging = ">=20.0.0" +pandas = {version = ">=1.1.0", optional = true, markers = "extra == \"pandas\""} +pyarrow = {version = ">=3.0.0", optional = true, markers = "extra == \"pandas\""} python-dateutil = ">=2.7.2,<3.0dev" requests = ">=2.21.0,<3.0.0dev" @@ -2838,4 +2841,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.9" -content-hash = "0def9d00f7a70c98da4e0d4ae2db848c68a9f56e6f17e532c398e87b373bd2d8" +content-hash = "a102f8b62ef85b71268f5d4b74ce5e65d37546f6bac49e308e497b0cd554e09c" diff --git a/profiles.yml b/profiles.yml index a00bcc17..82237d03 100644 --- a/profiles.yml +++ b/profiles.yml @@ -18,3 +18,12 @@ default: location: US threads: 2 keyfile: /credentials-prod/prod.json +elementary: + outputs: + default: + type: bigquery + project: basedosdados + dataset: elementary + method: service-account + keyfile: /tmp/bigquery_keyfile.json + threads: 2 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 18df2541..0e259aff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" [tool.poetry.dependencies] python = "~3.9" dbt-core = "1.5.6" -dbt-bigquery = "1.5.6" +dbt-bigquery = "1.5.9" dbt-rpc = "0.4.2" pyyaml = "6.0" From 8e7847ad1f41d80315d2e4e2f41487f0bacbd020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Mon, 1 Apr 2024 16:57:11 -0300 Subject: [PATCH 2/6] feat: add workflow dispatch trigger --- .github/workflows/{elementary.yml => elementary.yaml} | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) rename .github/workflows/{elementary.yml => elementary.yaml} (94%) diff --git a/.github/workflows/elementary.yml b/.github/workflows/elementary.yaml similarity index 94% rename from .github/workflows/elementary.yml rename to .github/workflows/elementary.yaml index 51242931..899a5cce 100644 --- a/.github/workflows/elementary.yml +++ b/.github/workflows/elementary.yaml @@ -1,8 +1,5 @@ name: Deploy Elementary Report -on: - push: - branches: [ "main", "master" ] - workflow_dispatch: +on: workflow_dispatch jobs: elementary: runs-on: ubuntu-latest From a3ed7bd9f5030074b4874c9d938c5672243313e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Mon, 1 Apr 2024 17:01:02 -0300 Subject: [PATCH 3/6] feat: add pull_request trigger --- .github/workflows/elementary.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/elementary.yaml b/.github/workflows/elementary.yaml index 899a5cce..3bdbed0a 100644 --- a/.github/workflows/elementary.yaml +++ b/.github/workflows/elementary.yaml @@ -1,5 +1,8 @@ name: Deploy Elementary Report -on: workflow_dispatch +on: + pull_request: + branches: [ "main", "master" ] + workflow_dispatch: jobs: elementary: runs-on: ubuntu-latest From e445161bd15a399552441e56893bf22521a76c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Mon, 1 Apr 2024 17:16:34 -0300 Subject: [PATCH 4/6] fix: change bucket name --- .github/workflows/elementary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elementary.yaml b/.github/workflows/elementary.yaml index 3bdbed0a..fdfd1bbc 100644 --- a/.github/workflows/elementary.yaml +++ b/.github/workflows/elementary.yaml @@ -21,7 +21,7 @@ jobs: && edr send-report --google-service-account-path "/tmp/gcs_keyfile.json" - --gcs-bucket-name "${{ secrets.GCS_BUCKET_NAME }}" + --gcs-bucket-name "basedosdados" --update-bucket-website "true" bigquery-keyfile: ${{ secrets.BIGQUERY_KEYFILE }} gcs-keyfile: ${{ secrets.GCS_KEYFILE }} From c099d5ad342b06eeb4766738cbc921bbc411efe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Mon, 1 Apr 2024 18:17:09 -0300 Subject: [PATCH 5/6] fix: rm workflow_dispatch --- .github/workflows/elementary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elementary.yaml b/.github/workflows/elementary.yaml index fdfd1bbc..cbc63f1e 100644 --- a/.github/workflows/elementary.yaml +++ b/.github/workflows/elementary.yaml @@ -1,5 +1,5 @@ name: Deploy Elementary Report -on: +on: pull_request: branches: [ "main", "master" ] workflow_dispatch: From 3a9c5c34942a45cee90b6ef9bbded6e4d0eb8fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Tue, 2 Apr 2024 18:44:59 -0300 Subject: [PATCH 6/6] feat: add cron schedule --- .github/workflows/elementary.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/elementary.yaml b/.github/workflows/elementary.yaml index cbc63f1e..1e93e8ab 100644 --- a/.github/workflows/elementary.yaml +++ b/.github/workflows/elementary.yaml @@ -1,7 +1,9 @@ name: Deploy Elementary Report on: - pull_request: + push: branches: [ "main", "master" ] + schedule: + - cron: '00 22 * * 1-5' workflow_dispatch: jobs: elementary: