Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[dbt] fix-improve br_me_comex_stat #558

Merged
merged 4 commits into from
Apr 24, 2024
Merged

[dbt] fix-improve br_me_comex_stat #558

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8c33894
feat: add br_me_comex_stat schema, models and macro
folhesgabriel Apr 24, 2024
bfe5e64
fix project name
folhesgabriel Apr 24, 2024
ab65095
make requested chagnes
folhesgabriel Apr 24, 2024
edf2b35
Merge branch 'main' into staging/fix_comex
folhesgabriel Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
297 changes: 297 additions & 0 deletions macros/transform_mdic_country_code.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,297 @@
{% macro transform_mdic_country_code(column_name) %}
-- mdic stands for: Ministério do Desenvolvimento, Indústria, Comércio e Serviços
-- it has its own country ids that can be found here:
-- https://balanca.economia.gov.br/balanca/bd/tabelas/PAIS.csv
-- NOTE:
-- '' values were set beacause the original matches were ZZZ which
-- indicates the inexistance of the code
case
{% for old_code, new_code in [
(0, null),
(13, "AFG"),
(15, "ALA"),
(17, "ALB"),
(20, "ESP"),
(23, "DEU"),
(25, "DEU"),
(31, "BFA"),
(37, "AND"),
(40, "AGO"),
(41, "AIA"),
(42, "ATA"),
(43, "ATG"),
(47, "ANT"),
(53, "SAU"),
(59, "DZA"),
(63, "ARG"),
(64, "ARM"),
(65, "ABW"),
(69, "AUS"),
(72, "AUT"),
(73, "AZE"),
(77, "BHS"),
(80, "BHR"),
(81, "BGD"),
(83, "BRB"),
(85, "BLR"),
(87, "BEL"),
(88, "BLZ"),
(90, "BMU"),
(93, "MMR"),
(97, "BOL"),
(98, "BIH"),
(99, "BES"),
(100, "BRA"),
(101, "BWA"),
(102, "BVT"),
(105, "BRA"),
(108, "BRN"),
(111, "BGR"),
(115, "BDI"),
(119, "BTN"),
(127, "CPV"),
(137, "CYM"),
(141, "KHM"),
(145, "CMR"),
(149, "CAN"),
(150, "GGY"),
(151, "ESP"),
(152, "GBR"),
(153, "KAZ"),
(154, "QAT"),
(158, "CHL"),
(160, "CHN"),
(161, "TWN"),
(163, "CYP"),
(165, "CCK"),
(169, "COL"),
(173, "COM"),
(177, "COG"),
(183, "COK"),
(187, "PRK"),
(190, "KOR"),
(193, "CIV"),
(195, "HRV"),
(196, "CRI"),
(198, "KWT"),
(199, "CUB"),
(200, "CUW"),
(229, "BEN"),
(232, "DNK"),
(235, "DMA"),
(237, "ARE"),
(239, "ECU"),
(240, "EGY"),
(243, "ERI"),
(244, "ARE"),
(245, "ESP"),
(246, "SVN"),
(247, "SVK"),
(249, "USA"),
(251, "EST"),
(253, "ETH"),
(255, "FLK"),
(259, "FRO"),
(267, "PHL"),
(271, "FIN"),
(275, "FRA"),
(281, "GAB"),
(285, "GMB"),
(289, "GHA"),
(291, "GEO"),
(292, "SGS"),
(293, "GIB"),
(297, "GRD"),
(301, "GRC"),
(305, "GRL"),
(309, "GLP"),
(313, "GUM"),
(317, "GTM"),
(321, "GGY"),
(325, "GUF"),
(329, "GIN"),
(331, "GNQ"),
(334, "GNB"),
(337, "GUY"),
(341, "HTI"),
(343, "HMD"),
(345, "HND"),
(351, "HKG"),
(355, "HUN"),
(357, "YEM"),
(358, "YMD"),
(359, "IMN"),
(361, "IND"),
(365, "IDN"),
(367, "GBR"),
(369, "IRQ"),
(372, "IRN"),
(375, "IRL"),
(379, "ISL"),
(383, "ISR"),
(386, "ITA"),
(388, "SCG"),
(391, "JAM"),
(393, null),
(396, "USA"),
(399, "JPN"),
(403, "JOR"),
(411, "KIR"),
(420, "LAO"),
(423, "MYS"),
(426, "LSO"),
(427, "LVA"),
(431, "LBN"),
(434, "LBR"),
(438, "LBY"),
(440, "LIE"),
(442, "LTU"),
(445, "LUX"),
(447, "MAC"),
(449, "MKD"),
(450, "MDG"),
(452, "PRT"),
(455, "MYS"),
(458, "MWI"),
(461, "MDV"),
(464, "MLI"),
(467, "MLT"),
(472, "MNP"),
(474, "MAR"),
(476, "MHL"),
(477, "MTQ"),
(485, "MUS"),
(488, "MRT"),
(489, null),
(490, "UMI"),
(493, "MEX"),
(494, "MDA"),
(495, "MCO"),
(497, "MNG"),
(498, "MNE"),
(499, "FSM"),
(501, "MSR"),
(505, "MOZ"),
(507, "NAM"),
(508, "NRU"),
(511, "CXR"),
(517, "NPL"),
(521, "NIC"),
(525, "NER"),
(528, "NGA"),
(531, "NIU"),
(535, "NFK"),
(538, "NOR"),
(542, "NCL"),
(545, "PNG"),
(548, "NZL"),
(551, "VUT"),
(556, "OMN"),
(563, "UMI"),
(566, "UMI"),
(569, "UMI"),
(573, "NLD"),
(575, "PLW"),
(576, "PAK"),
(578, "PSE"),
(580, "PAN"),
(583, "PNG"),
(586, "PRY"),
(589, "PER"),
(593, "PCN"),
(599, "PYF"),
(603, "POL"),
(607, "PRT"),
(611, "PRI"),
(623, "KEN"),
(625, "KGZ"),
(628, "GBR"),
(640, "CAF"),
(647, "DOM"),
(660, "REU"),
(665, "ZWE"),
(670, "ROU"),
(675, "RWA"),
(676, "RUS"),
(677, "SLB"),
(678, "KNA"),
(685, "ESH"),
(687, "SLV"),
(690, "WSM"),
(691, "ASM"),
(693, "BLM"),
(695, "KNA"),
(697, "SMR"),
(698, null),
(699, "SXM"),
(700, "SPM"),
(705, "VCT"),
(710, "SHN"),
(715, "LCA"),
(720, "STP"),
(728, "SEN"),
(731, "SYC"),
(735, "SLE"),
(737, "SRB"),
(741, "SGP"),
(744, "SYR"),
(748, "SOM"),
(750, "LKA"),
(754, "SWZ"),
(755, "SJM"),
(756, "ZAF"),
(759, "SDN"),
(760, "SSD"),
(764, "SWE"),
(767, "CHE"),
(770, "SUR"),
(772, "TJK"),
(776, "THA"),
(780, "TZA"),
(781, "ATF"),
(782, "IOT"),
(783, "DJI"),
(785, null),
(786, "GBR"),
(788, "TCD"),
(790, "CSK"),
(791, "CZE"),
(795, "TLS"),
(800, "TGO"),
(805, "TKL"),
(810, "TON"),
(815, "TTO"),
(820, "TUN"),
(823, "TCA"),
(824, "TKM"),
(827, "TUR"),
(828, "TUV"),
(831, "UKR"),
(833, "UGA"),
(840, null),
(845, "URY"),
(847, "UZB"),
(848, "VAT"),
(850, "VEN"),
(858, "VNM"),
(863, "VGB"),
(866, "VIR"),
(870, "FJI"),
(873, "USA"),
(875, "WLF"),
(888, "COD"),
(890, "ZMB"),
(895, "PCZ"),
(990, null),
(994, null),
(995, null),
(997, null),
(998, null),
(999, null),
] %}
when {{ column_name }} = '{{ old_code }}' then '{{ new_code }}'
{% endfor %}
else {{ column_name }}
end

{% endmacro %}
28 changes: 18 additions & 10 deletions models/br_me_comex_stat/br_me_comex_stat__municipio_exportacao.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,35 @@
config(
alias="municipio_exportacao",
schema="br_me_comex_stat",
materialized="table",
materialized="incremental",
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 1997, "end": 2023, "interval": 1},
"range": {"start": 1997, "end": 2025, "interval": 1},
},
cluster_by=["mes", "sigla_uf"],
labels={"project_id": "basedosdados", "tema": "economia"},
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)',
],
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
)
}}
select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_sh4 as string) id_sh4,
safe_cast(lpad(id_sh4, 4, '0') as string) id_sh4,
safe_cast(id_pais as string) id_pais,
safe_cast(sigla_uf as string) sigla_uf,
safe_cast(id_municipio as string) id_municipio,
{{ transform_mdic_country_code("id_pais") }} as sigla_pais_iso3,
safe_cast(case when sigla_uf = 'ND' then null else sigla_uf end as string) sigla_uf,
safe_cast(
case
when id_municipio = '9300000' or id_municipio = '9999999'
then null
else id_municipio
end as string
) id_municipio,
safe_cast(peso_liquido_kg as int64) peso_liquido_kg,
safe_cast(valor_fob_dolar as int64) valor_fob_dolar
from `basedosdados-staging.br_me_comex_stat_staging.municipio_exportacao` as t
{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
19 changes: 11 additions & 8 deletions models/br_me_comex_stat/br_me_comex_stat__municipio_importacao.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,27 @@
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 1997, "end": 2023, "interval": 1},
"range": {"start": 1997, "end": 2025, "interval": 1},
},
cluster_by=["mes", "sigla_uf"],
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)',
],
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_sh4 as string) id_sh4,
safe_cast(lpad(id_sh4, 4, '0') as string) id_sh4,
safe_cast(id_pais as string) id_pais,
safe_cast(sigla_uf as string) sigla_uf,
safe_cast(id_municipio as string) id_municipio,
{{ transform_mdic_country_code("id_pais") }} as sigla_pais_iso3,
safe_cast(case when sigla_uf = 'ND' then null else sigla_uf end as string) sigla_uf,
safe_cast(
case
when id_municipio = '9300000' or id_municipio = '9999999'
then null
else id_municipio
end as string
) id_municipio,
safe_cast(peso_liquido_kg as int64) peso_liquido_kg,
safe_cast(valor_fob_dolar as int64) valor_fob_dolar
from `basedosdados-staging.br_me_comex_stat_staging.municipio_importacao` as t
Expand Down
Loading
Loading