Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dbt: br_cgu_servidores_executivo_federal #288

Merged
merged 7 commits into from
Oct 24, 2023
Merged

dbt: br_cgu_servidores_executivo_federal #288

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5506d16
dbt: br_cgu_servidores_executivo_federal
aspeddro Oct 3, 2023
5e5397e
rename tables
aspeddro Oct 4, 2023
eb8e099
rename to staging
aspeddro Oct 4, 2023
f3907f3
fix col values
aspeddro Oct 5, 2023
8b7c746
fix col nivel_funcao
aspeddro Oct 5, 2023
4efe395
Merge branch 'main' into staging/br_cgu_servidores_executivo_federal
aspeddro Oct 24, 2023
fa70670
increase month
aspeddro Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ models:
+post-hook:
- 'REVOKE `roles/bigquery.dataViewer` ON TABLE {{ this }} FROM "specialGroup:allUsers"'
- 'GRANT `roles/bigquery.dataViewer` ON TABLE {{ this }} TO "group:[email protected]"'
br_cgu_servidores_executivo_federal:
+materialized: table
+schema: br_cgu_servidores_executivo_federal
br_cgu_pessoal_executivo_federal:
+materialized: table
+schema: br_cgu_pessoal_executivo_federal
Expand Down
51 changes: 51 additions & 0 deletions ...br_cgu_servidores_executivo_federal/br_cgu_servidores_executivo_federal__afastamentos.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
{{
config(
schema = 'br_cgu_servidores_executivo_federal',
alias = 'afastamentos',
materialized='table',
partition_by={
'field': 'ano',
'data_type': 'int64',
'range': {
"start": 2015,
"end": 2023,
"interval": 1
}
},
cluster_by=['ano', 'mes'],
post_hook = [
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter
ON {{this}}
GRANT TO ("allUsers")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter
ON {{this}}
GRANT TO ("group:[email protected]", "group:[email protected]")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'
]
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_servidor as string) id_servidor,
safe_cast(nome as string) nome,
safe_cast(cpf as string) cpf,
(
case
when data_inicio = "Não informada"
then null
else parse_date('%d/%m/%Y', data_inicio)
end
) as data_inicio,
(
case
when data_final = "Não informada"
then null
else parse_date('%d/%m/%Y', data_final)
end
) as data_final,
safe_cast(origem as string) origem,
from
`basedosdados-dev.br_cgu_servidores_executivo_federal_staging.afastamentos` as t
97 changes: 97 additions & 0 deletions ...ervidores_executivo_federal/br_cgu_servidores_executivo_federal__cadastro_aposentados.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
{{
config(
schema = 'br_cgu_servidores_executivo_federal',
alias = 'cadastro_aposentados',
materialized='table',
partition_by={
'field': 'ano',
'data_type': 'int64',
'range': {
"start": 2020,
"end": 2023,
"interval": 1
}
},
cluster_by=['ano', 'mes'],
post_hook = [
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter
ON {{this}}
GRANT TO ("allUsers")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter
ON {{this}}
GRANT TO ("group:[email protected]", "group:[email protected]")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'
]
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_servidor as string) id_servidor,
safe_cast(nome as string) nome,
safe_cast(cpf as string) cpf,
safe_cast(matricula as string) matricula,
safe_cast(id_tipo_aposentadoria as string) id_tipo_aposentadoria,
safe_cast(tipo_aposentadoria as string) tipo_aposentadoria,
(
case
when data_aposentadoria = "Não informada"
then null
else parse_date('%d/%m/%Y', data_aposentadoria)
end
) as data_aposentadoria,
laura-l-amaral marked this conversation as resolved.
Show resolved Hide resolved
safe_cast(descricao_cargo as string) descricao_cargo,
safe_cast(id_uorg_lotacao as string) id_uorg_lotacao,
safe_cast(uorg_lotacao as string) uorg_lotacao,
safe_cast(id_org_lotacao as string) id_org_lotacao,
safe_cast(org_lotacao as string) org_lotacao,
safe_cast(id_orgsup_lotacao as string) id_orgsup_lotacao,
safe_cast(orgsup_lotacao as string) orgsup_lotacao,
safe_cast(id_tipo_vinculo as string) id_tipo_vinculo,
safe_cast(tipo_vinculo as string) tipo_vinculo,
safe_cast(situacao_vinculo as string) situacao_vinculo,
safe_cast(regime_juridico as string) regime_juridico,
safe_cast(jornada_trabalho as string) jornada_trabalho,
(
case
when data_ingresso_cargo_funcao = "Não informada"
then null
else parse_date('%d/%m/%Y', data_ingresso_cargo_funcao)
end
) as data_ingresso_cargo_funcao,
(
case
when data_nomeacao_cargo_funcao = "Não informada"
then null
else parse_date('%d/%m/%Y', data_nomeacao_cargo_funcao)
end
) as data_nomeacao_cargo_funcao,
(
case
when data_ingresso_orgao = "Não informada"
then null
else parse_date('%d/%m/%Y', data_ingresso_orgao)
end
) as data_ingresso_orgao,
safe_cast(
documento_ingresso_servico_publico as string
) documento_ingresso_servico_publico,
(
case
when data_diploma_ingresso_servico_publico = "Não informada"
then null
else parse_date('%d/%m/%Y', data_diploma_ingresso_servico_publico)
end
) as data_diploma_ingresso_servico_publico,

safe_cast(diploma_ingresso_cargo_funcao as string) diploma_ingresso_cargo_funcao,
safe_cast(diploma_ingresso_orgao as string) diploma_ingresso_orgao,
safe_cast(
diploma_ingresso_servico_publico as string
) diploma_ingresso_servico_publico,
safe_cast(origem as string) origem,
from
`basedosdados-dev.br_cgu_servidores_executivo_federal_staging.cadastro_aposentados`
as t
110 changes: 110 additions & 0 deletions ...rvidores_executivo_federal/br_cgu_servidores_executivo_federal__cadastro_pensionistas.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
{{
config(
schema = 'br_cgu_servidores_executivo_federal',
alias = 'cadastro_pensionistas',
materialized='table',
partition_by={
'field': 'ano',
'data_type': 'int64',
'range': {
"start": 2020,
"end": 2023,
"interval": 1
}
},
cluster_by=['ano', 'mes'],
post_hook = [
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter
ON {{this}}
GRANT TO ("allUsers")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter
ON {{this}}
GRANT TO ("group:[email protected]", "group:[email protected]")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'
]
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_servidor as string) id_servidor,
safe_cast(nome as string) nome,
safe_cast(cpf as string) cpf,
safe_cast(matricula as string) matricula,
safe_cast(cpf_representante_legal as string) cpf_representante_legal,
safe_cast(nome_representante_legal as string) nome_representante_legal,
safe_cast(cpf_instituidor_pensao as string) cpf_instituidor_pensao,
safe_cast(nome_instituidor_pensao as string) nome_instituidor_pensao,
safe_cast(id_tipo_pensao as string) id_tipo_pensao,
safe_cast(tipo_pensao as string) tipo_pensao,
safe_cast(data_inicio_pensao as date) data_inicio_pensao,
safe_cast(
descricao_cargo_instituidor_pensao as string
) descricao_cargo_instituidor_pensao,
safe_cast(
id_uorg_lotacao_instituidor_pensao as string
) id_uorg_lotacao_instituidor_pensao,
safe_cast(
uorg_lotacao_instituidor_pensao as string
) uorg_lotacao_instituidor_pensao,
safe_cast(
id_org_lotacao_instituidor_pensao as string
) id_org_lotacao_instituidor_pensao,
safe_cast(org_lotacao_instituidor_pensao as string) org_lotacao_instituidor_pensao,
safe_cast(
id_orgsup_lotacao_instituidor_pensao as string
) id_orgsup_lotacao_instituidor_pensao,
safe_cast(
orgsup_lotacao_instituidor_pensao as string
) orgsup_lotacao_instituidor_pensao,
safe_cast(id_tipo_vinculo as string) id_tipo_vinculo,
safe_cast(tipo_vinculo as string) tipo_vinculo,
safe_cast(situacao_vinculo as string) situacao_vinculo,
safe_cast(
regime_juridico_instituidor_pensao as string
) regime_juridico_instituidor_pensao,
safe_cast(
jornada_trabalho_instituidor_pensao as string
) jornada_trabalho_instituidor_pensao,
(
case
when data_ingresso_cargo_funcao_instituidor_pensao = "Não informada" then null
else parse_date('%d/%m/%Y', data_ingresso_cargo_funcao_instituidor_pensao)
end
) as data_ingresso_cargo_funcao_instituidor_pensao,
(
case
when data_nomeacao_cargo_funcao_instituidor_pensao = "Não informada" then null
else parse_date('%d/%m/%Y', data_nomeacao_cargo_funcao_instituidor_pensao)
end
) as data_nomeacao_cargo_funcao_instituidor_pensao,
(
case
when data_ingresso_orgao_instituidor_pensao = "Não informada" then null
else parse_date('%d/%m/%Y', data_ingresso_orgao_instituidor_pensao)
end
) as data_ingresso_orgao_instituidor_pensao,
(
case
when data_diploma_ingresso_servico_publico_instituidor_pensao = "Não informada" then null
else parse_date('%d/%m/%Y', data_diploma_ingresso_servico_publico_instituidor_pensao)
end
) as data_diploma_ingresso_servico_publico_instituidor_pensao,
safe_cast(
documento_ingresso_servico_publico_instituidor_pensao as string
) documento_ingresso_servico_publico_instituidor_pensao,
safe_cast(
diploma_ingresso_cargo_funcao_instituidor_pensao as string
) diploma_ingresso_cargo_funcao_instituidor_pensao,
safe_cast(
diploma_ingresso_orgao_instituidor_pensao as string
) diploma_ingresso_orgao_instituidor_pensao,
safe_cast(
diploma_ingresso_servicopublico_instituidor_pensao as string
) diploma_ingresso_servicopublico_instituidor_pensao,
safe_cast(origem as string) origem,
from
`basedosdados-dev.br_cgu_servidores_executivo_federal_staging.cadastro_pensionistas`
as t
91 changes: 91 additions & 0 deletions ...utivo_federal/br_cgu_servidores_executivo_federal__cadastro_reserva_reforma_militares.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
{{
config(
schema = 'br_cgu_servidores_executivo_federal',
alias = 'cadastro_reserva_reforma_militares',
materialized='table',
partition_by={
'field': 'ano',
'data_type': 'int64',
'range': {
"start": 2020,
"end": 2023,
"interval": 1
}
},
cluster_by=['ano', 'mes'],
post_hook = [
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter
ON {{this}}
GRANT TO ("allUsers")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter
ON {{this}}
GRANT TO ("group:[email protected]", "group:[email protected]")
FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'
]
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_servidor as string) id_servidor,
safe_cast(nome as string) nome,
safe_cast(cpf as string) cpf,
safe_cast(matricula as string) matricula,
safe_cast(id_tipo_aposentadoria as string) id_tipo_aposentadoria,
safe_cast(tipo_aposentadoria as string) tipo_aposentadoria,
(
case
when data_aposentadoria = "Não informada" then null
else parse_date('%d/%m/%Y', data_aposentadoria)
end
) as data_aposentadoria,
safe_cast(descricao_cargo as string) descricao_cargo,
safe_cast(id_uorg_lotacao as string) id_uorg_lotacao,
safe_cast(uorg_lotacao as string) uorg_lotacao,
safe_cast(id_org_lotacao as string) id_org_lotacao,
safe_cast(org_lotacao as string) org_lotacao,
safe_cast(id_orgsup_lotacao as string) id_orgsup_lotacao,
safe_cast(orgsup_lotacao as string) orgsup_lotacao,
safe_cast(id_tipo_vinculo as string) id_tipo_vinculo,
safe_cast(tipo_vinculo as string) tipo_vinculo,
safe_cast(situacao_vinculo as string) situacao_vinculo,
safe_cast(regime_juridico as string) regime_juridico,
safe_cast(jornada_trabalho as string) jornada_trabalho,
(
case
when data_ingresso_cargo_funcao = "Não informada" then null
else parse_date('%d/%m/%Y', data_ingresso_cargo_funcao)
end
) as data_ingresso_cargo_funcao,
(
case
when data_nomeacao_cargo_funcao = "Não informada" then null
else parse_date('%d/%m/%Y', data_nomeacao_cargo_funcao)
end
) as data_nomeacao_cargo_funcao,
(
case
when data_ingresso_orgao = "Não informada" then null
else parse_date('%d/%m/%Y', data_ingresso_orgao)
end
) as data_ingresso_orgao,
(
case
when data_diploma_ingresso_servico_publico = "Não informada" then null
else parse_date('%d/%m/%Y', data_diploma_ingresso_servico_publico)
end
) as data_diploma_ingresso_servico_publico,
safe_cast(
documento_ingresso_servico_publico as string
) documento_ingresso_servico_publico,
safe_cast(diploma_ingresso_cargo_funcao as string) diploma_ingresso_cargo_funcao,
safe_cast(diploma_ingresso_orgao as string) diploma_ingresso_orgao,
safe_cast(
diploma_ingresso_servico_publico as string
) diploma_ingresso_servico_publico,
from
`basedosdados-dev.br_cgu_servidores_executivo_federal_staging.cadastro_reserva_reforma_militares`
as t

Loading