From 3c28c01918a0ce02c5b48d4a3b77dd549eb72415 Mon Sep 17 00:00:00 2001 From: tricktx Date: Wed, 6 Sep 2023 21:47:15 -0300 Subject: [PATCH 1/3] model incremental --- .../br_mg_belohorizonte_smfa_iptu__iptu.sql | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql index 4b95ad6e..c004af61 100644 --- a/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql +++ b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql @@ -3,14 +3,16 @@ schema='br_mg_belohorizonte_smfa_iptu', materialized='incremental', partition_by={ - "field": "mes", + "field": "ano", "data_type": "int64", - "granularity": "month"}, + "range": { + "start": 2022, + "end": 2023, + "interval": 1} + }, cluster_by=['tipo_construtivo', 'tipo_ocupacao', 'tipologia'], labels = {'project_id' : 'basedosdados-dev'} -) }} - -WITH iptu AS ( +)}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -37,12 +39,8 @@ SAFE_CAST(indicador_agua AS BOOL) indicador_agua, SAFE.ST_GEOGFROMTEXT(poligono) poligono, SAFE_CAST(fracao_ideal AS FLOAT64) fracao_ideal, SAFE_CAST(area_terreno AS FLOAT64) area_terreno, -SAFE_CAST(area_construida AS FLOAT64) area_construida, -FROM basedosdados-dev.br_mg_belohorizonte_smfa_iptu_staging.iptu AS t) -SELECT * FROM iptu - -{% if is_incremental() %} - -WHERE ano > (SELECT max(ano) FROM {{ this }}) and mes > (SELECT max(mes) FROM {{ this }}) - +SAFE_CAST(area_construida AS FLOAT64) area_construida +FROM basedosdados-dev.br_mg_belohorizonte_smfa_iptu_staging.iptu AS t +{% if is_incremental() %} +WHERE CONCAT(ano,mes) > (SELECT MAX(CONCAT(ano,mes)) FROM {{ this }} ) {% endif %} \ No newline at end of file From 6c87a0ae9651f3d471c4f7f55f20766a7e6db5c4 Mon Sep 17 00:00:00 2001 From: tricktx Date: Tue, 12 Sep 2023 09:26:46 -0300 Subject: [PATCH 2/3] add ano and mes --- .../br_mg_belohorizonte_smfa_iptu__iptu.sql | 8 ++++---- models/br_mg_belohorizonte_smfa_iptu/schema.yml | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql index c004af61..f735ba6d 100644 --- a/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql +++ b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__iptu.sql @@ -1,7 +1,7 @@ {{ config( alias='iptu', schema='br_mg_belohorizonte_smfa_iptu', - materialized='incremental', + materialized='table', partition_by={ "field": "ano", "data_type": "int64", @@ -41,6 +41,6 @@ SAFE_CAST(fracao_ideal AS FLOAT64) fracao_ideal, SAFE_CAST(area_terreno AS FLOAT64) area_terreno, SAFE_CAST(area_construida AS FLOAT64) area_construida FROM basedosdados-dev.br_mg_belohorizonte_smfa_iptu_staging.iptu AS t -{% if is_incremental() %} -WHERE CONCAT(ano,mes) > (SELECT MAX(CONCAT(ano,mes)) FROM {{ this }} ) -{% endif %} \ No newline at end of file +-- {% if is_incremental() %} +-- WHERE CONCAT(ano,mes) > (SELECT MAX(CONCAT(ano,mes)) FROM {{ this }} ) +-- {% endif %} \ No newline at end of file diff --git a/models/br_mg_belohorizonte_smfa_iptu/schema.yml b/models/br_mg_belohorizonte_smfa_iptu/schema.yml index f2192020..40209f5f 100644 --- a/models/br_mg_belohorizonte_smfa_iptu/schema.yml +++ b/models/br_mg_belohorizonte_smfa_iptu/schema.yml @@ -1,9 +1,13 @@ version: 2 models: - - name: br_mg_belohorizonte_smfa_iptu_iptu + - name: br_mg_belohorizonte_smfa_iptu__iptu description: Dados referente ao cadastro do IPTU na cidade de Belo Horizonte columns: + - name: ano + description: Ano + - name: mes + description: Mês - name: indice_cadastral description: Os imóveis, para inscrição no Cadastro Imobiliário, foram codificados através de sua associação a um código alfanumérico denominado índice cadastral, composto de 15 dígitos. - name: lote From 22e837e4501a3c10d5bca0391ee939482419f270 Mon Sep 17 00:00:00 2001 From: tricktx Date: Tue, 12 Sep 2023 09:48:52 -0300 Subject: [PATCH 3/3] add dicionario --- ...br_mg_belohorizonte_smfa_iptu__dicionario.sql | 10 ++++++++++ models/br_mg_belohorizonte_smfa_iptu/schema.yml | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__dicionario.sql diff --git a/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__dicionario.sql b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__dicionario.sql new file mode 100644 index 00000000..4e1de3b9 --- /dev/null +++ b/models/br_mg_belohorizonte_smfa_iptu/br_mg_belohorizonte_smfa_iptu__dicionario.sql @@ -0,0 +1,10 @@ +{{ config(alias='dicionario', schema='br_mg_belohorizonte_smfa_iptu') }} + + +SELECT +SAFE_CAST(id_tabela AS STRING) id_tabela, +SAFE_CAST(nome_coluna AS STRING) nome_coluna, +SAFE_CAST(chave AS STRING) chave, +SAFE_CAST(cobertura_temporal AS STRING) cobertura_temporal, +SAFE_CAST(valor AS STRING) valor +FROM basedosdados-dev.br_mg_belohorizonte_smfa_iptu_staging.dicionario AS t \ No newline at end of file diff --git a/models/br_mg_belohorizonte_smfa_iptu/schema.yml b/models/br_mg_belohorizonte_smfa_iptu/schema.yml index 40209f5f..ac79e009 100644 --- a/models/br_mg_belohorizonte_smfa_iptu/schema.yml +++ b/models/br_mg_belohorizonte_smfa_iptu/schema.yml @@ -55,4 +55,18 @@ models: - name: area_terreno description: Área do terreno - name: area_construida - description: Área construida \ No newline at end of file + description: Área construida + + - name: br_mg_belohorizonte_smfa_iptu__iptu_dicionario + description: "Dicionário de dados da tabela de IPTU da cidade de Belo Horizonte" + columns: + - name: id_coluna + description: ID da coluna + - name: nome_coluna + description: Nome da coluna + - name: chave + description: Chave + - name: cobertura_temporal + description: Cobertura temporal + - name: valor + description: Valor \ No newline at end of file