Skip to content

Commit

Permalink
Merge pull request #558 from basedosdados/staging/fix_comex
Browse files Browse the repository at this point in the history 
[dbt] fix-improve br_me_comex_stat
  • Loading branch information
@folhesgabriel
folhesgabriel authored Apr 24, 2024
2 parents 66e2791 + edf2b35 commit 1b47a6c
Show file tree
Hide file tree
Showing 6 changed files with 447 additions and 42 deletions.
297 changes: 297 additions & 0 deletions macros/transform_mdic_country_code.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,297 @@
{% macro transform_mdic_country_code(column_name) %}
-- mdic stands for: Ministério do Desenvolvimento, Indústria, Comércio e Serviços
-- it has its own country ids that can be found here:
-- https://balanca.economia.gov.br/balanca/bd/tabelas/PAIS.csv
-- NOTE:
-- '' values were set beacause the original matches were ZZZ which
-- indicates the inexistance of the code
case
{% for old_code, new_code in [
(0, null),
(13, "AFG"),
(15, "ALA"),
(17, "ALB"),
(20, "ESP"),
(23, "DEU"),
(25, "DEU"),
(31, "BFA"),
(37, "AND"),
(40, "AGO"),
(41, "AIA"),
(42, "ATA"),
(43, "ATG"),
(47, "ANT"),
(53, "SAU"),
(59, "DZA"),
(63, "ARG"),
(64, "ARM"),
(65, "ABW"),
(69, "AUS"),
(72, "AUT"),
(73, "AZE"),
(77, "BHS"),
(80, "BHR"),
(81, "BGD"),
(83, "BRB"),
(85, "BLR"),
(87, "BEL"),
(88, "BLZ"),
(90, "BMU"),
(93, "MMR"),
(97, "BOL"),
(98, "BIH"),
(99, "BES"),
(100, "BRA"),
(101, "BWA"),
(102, "BVT"),
(105, "BRA"),
(108, "BRN"),
(111, "BGR"),
(115, "BDI"),
(119, "BTN"),
(127, "CPV"),
(137, "CYM"),
(141, "KHM"),
(145, "CMR"),
(149, "CAN"),
(150, "GGY"),
(151, "ESP"),
(152, "GBR"),
(153, "KAZ"),
(154, "QAT"),
(158, "CHL"),
(160, "CHN"),
(161, "TWN"),
(163, "CYP"),
(165, "CCK"),
(169, "COL"),
(173, "COM"),
(177, "COG"),
(183, "COK"),
(187, "PRK"),
(190, "KOR"),
(193, "CIV"),
(195, "HRV"),
(196, "CRI"),
(198, "KWT"),
(199, "CUB"),
(200, "CUW"),
(229, "BEN"),
(232, "DNK"),
(235, "DMA"),
(237, "ARE"),
(239, "ECU"),
(240, "EGY"),
(243, "ERI"),
(244, "ARE"),
(245, "ESP"),
(246, "SVN"),
(247, "SVK"),
(249, "USA"),
(251, "EST"),
(253, "ETH"),
(255, "FLK"),
(259, "FRO"),
(267, "PHL"),
(271, "FIN"),
(275, "FRA"),
(281, "GAB"),
(285, "GMB"),
(289, "GHA"),
(291, "GEO"),
(292, "SGS"),
(293, "GIB"),
(297, "GRD"),
(301, "GRC"),
(305, "GRL"),
(309, "GLP"),
(313, "GUM"),
(317, "GTM"),
(321, "GGY"),
(325, "GUF"),
(329, "GIN"),
(331, "GNQ"),
(334, "GNB"),
(337, "GUY"),
(341, "HTI"),
(343, "HMD"),
(345, "HND"),
(351, "HKG"),
(355, "HUN"),
(357, "YEM"),
(358, "YMD"),
(359, "IMN"),
(361, "IND"),
(365, "IDN"),
(367, "GBR"),
(369, "IRQ"),
(372, "IRN"),
(375, "IRL"),
(379, "ISL"),
(383, "ISR"),
(386, "ITA"),
(388, "SCG"),
(391, "JAM"),
(393, null),
(396, "USA"),
(399, "JPN"),
(403, "JOR"),
(411, "KIR"),
(420, "LAO"),
(423, "MYS"),
(426, "LSO"),
(427, "LVA"),
(431, "LBN"),
(434, "LBR"),
(438, "LBY"),
(440, "LIE"),
(442, "LTU"),
(445, "LUX"),
(447, "MAC"),
(449, "MKD"),
(450, "MDG"),
(452, "PRT"),
(455, "MYS"),
(458, "MWI"),
(461, "MDV"),
(464, "MLI"),
(467, "MLT"),
(472, "MNP"),
(474, "MAR"),
(476, "MHL"),
(477, "MTQ"),
(485, "MUS"),
(488, "MRT"),
(489, null),
(490, "UMI"),
(493, "MEX"),
(494, "MDA"),
(495, "MCO"),
(497, "MNG"),
(498, "MNE"),
(499, "FSM"),
(501, "MSR"),
(505, "MOZ"),
(507, "NAM"),
(508, "NRU"),
(511, "CXR"),
(517, "NPL"),
(521, "NIC"),
(525, "NER"),
(528, "NGA"),
(531, "NIU"),
(535, "NFK"),
(538, "NOR"),
(542, "NCL"),
(545, "PNG"),
(548, "NZL"),
(551, "VUT"),
(556, "OMN"),
(563, "UMI"),
(566, "UMI"),
(569, "UMI"),
(573, "NLD"),
(575, "PLW"),
(576, "PAK"),
(578, "PSE"),
(580, "PAN"),
(583, "PNG"),
(586, "PRY"),
(589, "PER"),
(593, "PCN"),
(599, "PYF"),
(603, "POL"),
(607, "PRT"),
(611, "PRI"),
(623, "KEN"),
(625, "KGZ"),
(628, "GBR"),
(640, "CAF"),
(647, "DOM"),
(660, "REU"),
(665, "ZWE"),
(670, "ROU"),
(675, "RWA"),
(676, "RUS"),
(677, "SLB"),
(678, "KNA"),
(685, "ESH"),
(687, "SLV"),
(690, "WSM"),
(691, "ASM"),
(693, "BLM"),
(695, "KNA"),
(697, "SMR"),
(698, null),
(699, "SXM"),
(700, "SPM"),
(705, "VCT"),
(710, "SHN"),
(715, "LCA"),
(720, "STP"),
(728, "SEN"),
(731, "SYC"),
(735, "SLE"),
(737, "SRB"),
(741, "SGP"),
(744, "SYR"),
(748, "SOM"),
(750, "LKA"),
(754, "SWZ"),
(755, "SJM"),
(756, "ZAF"),
(759, "SDN"),
(760, "SSD"),
(764, "SWE"),
(767, "CHE"),
(770, "SUR"),
(772, "TJK"),
(776, "THA"),
(780, "TZA"),
(781, "ATF"),
(782, "IOT"),
(783, "DJI"),
(785, null),
(786, "GBR"),
(788, "TCD"),
(790, "CSK"),
(791, "CZE"),
(795, "TLS"),
(800, "TGO"),
(805, "TKL"),
(810, "TON"),
(815, "TTO"),
(820, "TUN"),
(823, "TCA"),
(824, "TKM"),
(827, "TUR"),
(828, "TUV"),
(831, "UKR"),
(833, "UGA"),
(840, null),
(845, "URY"),
(847, "UZB"),
(848, "VAT"),
(850, "VEN"),
(858, "VNM"),
(863, "VGB"),
(866, "VIR"),
(870, "FJI"),
(873, "USA"),
(875, "WLF"),
(888, "COD"),
(890, "ZMB"),
(895, "PCZ"),
(990, null),
(994, null),
(995, null),
(997, null),
(998, null),
(999, null),
] %}
when {{ column_name }} = '{{ old_code }}' then '{{ new_code }}'
{% endfor %}
else {{ column_name }}
end

{% endmacro %}
28 changes: 18 additions & 10 deletions models/br_me_comex_stat/br_me_comex_stat__municipio_exportacao.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,35 @@
config(
alias="municipio_exportacao",
schema="br_me_comex_stat",
materialized="table",
materialized="incremental",
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 1997, "end": 2023, "interval": 1},
"range": {"start": 1997, "end": 2025, "interval": 1},
},
cluster_by=["mes", "sigla_uf"],
labels={"project_id": "basedosdados", "tema": "economia"},
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)',
],
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
)
}}
select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_sh4 as string) id_sh4,
safe_cast(lpad(id_sh4, 4, '0') as string) id_sh4,
safe_cast(id_pais as string) id_pais,
safe_cast(sigla_uf as string) sigla_uf,
safe_cast(id_municipio as string) id_municipio,
{{ transform_mdic_country_code("id_pais") }} as sigla_pais_iso3,
safe_cast(case when sigla_uf = 'ND' then null else sigla_uf end as string) sigla_uf,
safe_cast(
case
when id_municipio = '9300000' or id_municipio = '9999999'
then null
else id_municipio
end as string
) id_municipio,
safe_cast(peso_liquido_kg as int64) peso_liquido_kg,
safe_cast(valor_fob_dolar as int64) valor_fob_dolar
from `basedosdados-staging.br_me_comex_stat_staging.municipio_exportacao` as t
{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
19 changes: 11 additions & 8 deletions models/br_me_comex_stat/br_me_comex_stat__municipio_importacao.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,27 @@
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 1997, "end": 2023, "interval": 1},
"range": {"start": 1997, "end": 2025, "interval": 1},
},
cluster_by=["mes", "sigla_uf"],
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 0)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 6 OR DATE_DIFF(DATE(2023,5,1),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) < 0)',
],
)
}}

select
safe_cast(ano as int64) ano,
safe_cast(mes as int64) mes,
safe_cast(id_sh4 as string) id_sh4,
safe_cast(lpad(id_sh4, 4, '0') as string) id_sh4,
safe_cast(id_pais as string) id_pais,
safe_cast(sigla_uf as string) sigla_uf,
safe_cast(id_municipio as string) id_municipio,
{{ transform_mdic_country_code("id_pais") }} as sigla_pais_iso3,
safe_cast(case when sigla_uf = 'ND' then null else sigla_uf end as string) sigla_uf,
safe_cast(
case
when id_municipio = '9300000' or id_municipio = '9999999'
then null
else id_municipio
end as string
) id_municipio,
safe_cast(peso_liquido_kg as int64) peso_liquido_kg,
safe_cast(valor_fob_dolar as int64) valor_fob_dolar
from `basedosdados-staging.br_me_comex_stat_staging.municipio_importacao` as t
Expand Down
Loading

0 comments on commit 1b47a6c

Please sign in to comment.