From 0ece6767274cd10487de2c75ea9bfcc1c5416042 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Wed, 20 Mar 2024 13:47:02 -0300 Subject: [PATCH 1/4] feat: convert bytes to string column tipo_avaliacao_pnass --- models/br_ms_cnes/br_ms_cnes__estabelecimento.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql index 4db4951b..42551708 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql @@ -96,7 +96,9 @@ select safe_cast(clasaval as string) tipo_classificacao_acreditacao_hospitalar, safe_cast(substr(cast(dt_acred as string), 1, 4) as int64) as ano_acreditacao, safe_cast(substr(cast(dt_acred as string), 5, 2) as int64) as mes_acreditacao, - safe_cast({{ clean_cols("AV_PNASS") }} as int64) tipo_avaliacao_pnass, + safe_cast( + cast({{ clean_cols("AV_PNASS") }} as string) as int64 + ) tipo_avaliacao_pnass, safe_cast(substr(cast(dt_pnass as string), 1, 4) as int64) as ano_avaliacao_pnass, safe_cast(substr(cast(dt_pnass as string), 5, 2) as int64) as mes_avaliacao_pnass, safe_cast(nivate_a as int64) indicador_atencao_ambulatorial, From 2d6a0a0219b0c877cfa223f0ee89567bdf19e690 Mon Sep 17 00:00:00 2001 From: tricktx Date: Thu, 21 Mar 2024 18:03:02 -0300 Subject: [PATCH 2/4] fix max data --- ...a_dados_abertos__proposicao_microdados.sql | 116 ++++++++++++------ 1 file changed, 80 insertions(+), 36 deletions(-) diff --git a/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql b/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql index f99989ff..402e472e 100644 --- a/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql +++ b/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql @@ -6,43 +6,87 @@ partition_by={ "field": "ano", "data_type": "INT64", - "range": {"start": 1935, "end": 2023, "interval": 1}, + "range": {"start": 1935, "end": 2024, "interval": 1}, }, ) }} -select - safe_cast(ano as int64) ano, - safe_cast( - split( - format_timestamp('%Y-%m-%dT%H:%M:%E*S', timestamp(dataapresentacao)), 'T' - )[offset(0)] as date - ) data, - safe_cast( - split( - format_timestamp('%Y-%m-%dT%H:%M:%E*S', timestamp(dataapresentacao)), 'T' - )[offset(1)] as time - ) horario, - safe_cast(id as string) id_proposicao, - safe_cast(uri as string) url, - safe_cast(numero as string) numero, - safe_cast(siglatipo as string) sigla, - safe_cast(descricaotipo as string) tipo, - safe_cast(ementa as string) ementa, - safe_cast(ementadetalhada as string) ementa_detalhada, - safe_cast(keywords as string) palavra_chave, - safe_cast(uriorgaonumerador as string) url_orgao_numerador, - safe_cast(uripropprincipal as string) url_principal, - safe_cast(uripropposterior as string) url_posterior, - safe_cast(urlinteiroteor as string) url_teor_proposicao, - safe_cast(ultimostatus_datahora as string) data_hora_ultimo_status, - safe_cast(ultimostatus_urirelator as string) url_relator_ultimo_status, - safe_cast(ultimostatus_siglaorgao as string) sigla_orgao_ultimo_status, - safe_cast(ultimostatus_regime as string) regime_ultimo_status, - safe_cast(ultimostatus_descricaotramitacao as string) tramitacao_ultimo_status, - safe_cast(ultimostatus_descricaosituacao as string) situacao_ultimo_status, - safe_cast(ultimostatus_despacho as string) despacho_ultimo_status, - safe_cast(ultimostatus_apreciacao as string) apreciacao_ultimo_status, - safe_cast(ultimostatus_sequencia as string) sequencia_ultimo_status, - safe_cast(ultimostatus_url as string) url_ultimo_status, -from `basedosdados-staging.br_camara_dados_abertos_staging.proposicao_microdados` as t +with + table as ( + select + safe_cast(ano as int64) ano, + safe_cast( + split( + format_timestamp( + '%Y-%m-%dT%H:%M:%E*S', timestamp(dataapresentacao) + ), + 'T' + )[offset (0)] as date + ) data, + safe_cast( + split( + format_timestamp( + '%Y-%m-%dT%H:%M:%E*S', timestamp(dataapresentacao) + ), + 'T' + )[offset (1)] as time + ) horario, + safe_cast(id as string) id_proposicao, + safe_cast(uri as string) url, + safe_cast(numero as string) numero, + safe_cast(siglatipo as string) sigla, + safe_cast(descricaotipo as string) tipo, + safe_cast(ementa as string) ementa, + safe_cast(ementadetalhada as string) ementa_detalhada, + safe_cast(keywords as string) palavra_chave, + safe_cast(uriorgaonumerador as string) url_orgao_numerador, + safe_cast(uripropprincipal as string) url_principal, + safe_cast(uripropposterior as string) url_posterior, + safe_cast(urlinteiroteor as string) url_teor_proposicao, + safe_cast(ultimostatus_datahora as string) data_hora_ultimo_status, + safe_cast(ultimostatus_urirelator as string) url_relator_ultimo_status, + safe_cast(ultimostatus_siglaorgao as string) sigla_orgao_ultimo_status, + safe_cast(ultimostatus_regime as string) regime_ultimo_status, + safe_cast( + ultimostatus_descricaotramitacao as string + ) tramitacao_ultimo_status, + safe_cast(ultimostatus_descricaosituacao as string) situacao_ultimo_status, + safe_cast(ultimostatus_despacho as string) despacho_ultimo_status, + safe_cast(ultimostatus_apreciacao as string) apreciacao_ultimo_status, + safe_cast(ultimostatus_sequencia as string) sequencia_ultimo_status, + safe_cast(ultimostatus_url as string) url_ultimo_status, + from + `basedosdados-dev.br_camara_dados_abertos_staging.proposicao_microdados` + as t + ), + query_total as ( + select + ano, + case when data >= current_date() then null else data end as data, + horario, + id_proposicao, + url, + numero, + sigla, + tipo, + ementa, + ementa_detalhada, + palavra_chave, + url_orgao_numerador, + url_principal, + url_posterior, + url_teor_proposicao, + data_hora_ultimo_status, + url_relator_ultimo_status, + sigla_orgao_ultimo_status, + regime_ultimo_status, + tramitacao_ultimo_status, + situacao_ultimo_status, + despacho_ultimo_status, + apreciacao_ultimo_status, + sequencia_ultimo_status, + url_ultimo_status, + from table + ) +select * +from query_total From 7c21c2c43e1610708e2042373d74dba38b68994a Mon Sep 17 00:00:00 2001 From: tricktx Date: Fri, 22 Mar 2024 15:41:15 -0300 Subject: [PATCH 3/4] fix wrong line --- ...r_camara_dados_abertos__orgao_deputado.sql | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/models/br_camara_dados_abertos/br_camara_dados_abertos__orgao_deputado.sql b/models/br_camara_dados_abertos/br_camara_dados_abertos__orgao_deputado.sql index f5f82eda..a1770fe0 100644 --- a/models/br_camara_dados_abertos/br_camara_dados_abertos__orgao_deputado.sql +++ b/models/br_camara_dados_abertos/br_camara_dados_abertos__orgao_deputado.sql @@ -1,12 +1,25 @@ {{ config(alias="orgao_deputado", schema="br_camara_dados_abertos") }} -select distinct - regexp_extract(uriorgao, r'/orgaos/(\d+)') as id_orgao, - safe_cast(nomeorgao as string) nome, - safe_cast(siglaorgao as string) sigla, - safe_cast(nomedeputado as string) nome_deputado, - safe_cast(cargo as string) cargo, - safe_cast(siglauf as string) sigla_uf, - safe_cast(datainicio as date) data_inicio, - safe_cast(datafim as date) data_final, - safe_cast(siglapartido as string) sigla_partido, -from `basedosdados-staging.br_camara_dados_abertos_staging.orgao_deputado` as t +with + orgao_deputado as ( + select distinct + regexp_extract(uriorgao, r'/orgaos/(\d+)') as id_orgao, + safe_cast(nomeorgao as string) nome, + safe_cast(siglaorgao as string) sigla, + safe_cast(nomedeputado as string) nome_deputado, + safe_cast(cargo as string) cargo, + safe_cast(siglauf as string) sigla_uf, + safe_cast(datainicio as date) data_inicio, + safe_cast(datafim as date) data_final, + safe_cast(siglapartido as string) sigla_partido, + from `basedosdados-staging.br_camara_dados_abertos_staging.orgao_deputado` + ) +select * +from orgao_deputado +where + not ( + nome_deputado = 'Hélio Leite' + and cargo = 'Titular' + and sigla_uf is null + and data_inicio = '2022-05-03' + and data_final = '2023-02-01' + ) From 375aafe50c30836ec8d2b7c147dbe4ea8a640c01 Mon Sep 17 00:00:00 2001 From: tricktx Date: Fri, 22 Mar 2024 16:40:53 -0300 Subject: [PATCH 4/4] fix dev --- .../br_camara_dados_abertos__proposicao_microdados.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql b/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql index 402e472e..359ae7c5 100644 --- a/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql +++ b/models/br_camara_dados_abertos/br_camara_dados_abertos__proposicao_microdados.sql @@ -56,7 +56,7 @@ with safe_cast(ultimostatus_sequencia as string) sequencia_ultimo_status, safe_cast(ultimostatus_url as string) url_ultimo_status, from - `basedosdados-dev.br_camara_dados_abertos_staging.proposicao_microdados` + `basedosdados-staging.br_camara_dados_abertos_staging.proposicao_microdados` as t ), query_total as (