From 8abb2123eec689dee2edd9449f2231b540407158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arthur=20Gusm=C3=A3o?= Date: Wed, 11 Oct 2023 12:45:57 -0300 Subject: [PATCH] fix: fixing horas_contratuais --- models/br_me_caged/microdados_movimentacao.sql | 10 +++++++--- .../br_me_caged/microdados_movimentacao_excluida.sql | 10 +++++++--- .../br_me_caged/microdados_movimentacao_fora_prazo.sql | 10 +++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/models/br_me_caged/microdados_movimentacao.sql b/models/br_me_caged/microdados_movimentacao.sql index 9f9fb8a4..46def14c 100644 --- a/models/br_me_caged/microdados_movimentacao.sql +++ b/models/br_me_caged/microdados_movimentacao.sql @@ -12,15 +12,16 @@ }, cluster_by = ["mes", "sigla_uf"], labels = {'project_id': 'basedosdados', 'tema': 'economia'}, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook = [ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) }} SELECT SAFE_CAST(ano AS INT64) ano, @@ -34,7 +35,10 @@ SAFE_CAST(cbo_2002 AS STRING) cbo_2002, SAFE_CAST(categoria AS STRING) categoria, SAFE_CAST(grau_instrucao AS STRING) grau_instrucao, SAFE_CAST(REPLACE(idade,'.0','') AS INT64) idade, -SAFE_CAST(REPLACE(horas_contratuais,',00','') AS INT64) horas_contratuais, +CASE + WHEN horas_contratuais LIKE '%,%' THEN CAST(REPLACE(horas_contratuais, ',', '.') AS FLOAT64) + ELSE CAST(horas_contratuais AS FLOAT64) +END AS horas_contratuais, SAFE_CAST(raca_cor AS STRING) raca_cor, SAFE_CAST(sexo AS STRING) sexo, SAFE_CAST(tipo_empregador AS STRING) tipo_empregador, diff --git a/models/br_me_caged/microdados_movimentacao_excluida.sql b/models/br_me_caged/microdados_movimentacao_excluida.sql index d96c2bf5..a7acf951 100644 --- a/models/br_me_caged/microdados_movimentacao_excluida.sql +++ b/models/br_me_caged/microdados_movimentacao_excluida.sql @@ -12,15 +12,16 @@ }, cluster_by = ["mes", "sigla_uf"], labels = {'project_id': 'basedosdados', 'tema': 'economia'}, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook = [ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) }} SELECT SAFE_CAST(ano AS INT64) ano, @@ -34,7 +35,10 @@ SAFE_CAST(cbo_2002 AS STRING) cbo_2002, SAFE_CAST(categoria AS STRING) categoria, SAFE_CAST(grau_instrucao AS STRING) grau_instrucao, SAFE_CAST(REPLACE(idade,'.0','') AS INT64) idade, -SAFE_CAST(REPLACE(horas_contratuais,',00','') AS INT64) horas_contratuais, +CASE + WHEN horas_contratuais LIKE '%,%' THEN CAST(REPLACE(horas_contratuais, ',', '.') AS FLOAT64) + ELSE CAST(horas_contratuais AS FLOAT64) +END AS horas_contratuais, SAFE_CAST(raca_cor AS STRING) raca_cor, SAFE_CAST(sexo AS STRING) sexo, SAFE_CAST(tipo_empregador AS STRING) tipo_empregador, diff --git a/models/br_me_caged/microdados_movimentacao_fora_prazo.sql b/models/br_me_caged/microdados_movimentacao_fora_prazo.sql index 6284d7a1..f0acf53a 100644 --- a/models/br_me_caged/microdados_movimentacao_fora_prazo.sql +++ b/models/br_me_caged/microdados_movimentacao_fora_prazo.sql @@ -12,15 +12,16 @@ }, cluster_by = ["mes", "sigla_uf"], labels = {'project_id': 'basedosdados', 'tema': 'economia'}, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook = [ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)', 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) + FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)'] ) }} SELECT SAFE_CAST(ano AS INT64) ano, @@ -34,7 +35,10 @@ SAFE_CAST(cbo_2002 AS STRING) cbo_2002, SAFE_CAST(categoria AS STRING) categoria, SAFE_CAST(grau_instrucao AS STRING) grau_instrucao, SAFE_CAST(REPLACE(idade,'.0','') AS INT64) idade, -SAFE_CAST(REPLACE(horas_contratuais,',00','') AS INT64) horas_contratuais, +CASE + WHEN horas_contratuais LIKE '%,%' THEN CAST(REPLACE(horas_contratuais, ',', '.') AS FLOAT64) + ELSE CAST(horas_contratuais AS FLOAT64) +END AS horas_contratuais, SAFE_CAST(raca_cor AS STRING) raca_cor, SAFE_CAST(sexo AS STRING) sexo, SAFE_CAST(tipo_empregador AS STRING) tipo_empregador,