Skip to content

Commit

Permalink
Merge branch 'main' into pre-commit-ci-update-config
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] authored Aug 8, 2024
2 parents 0498f84 + c8eb4e7 commit 08c37bb
Show file tree
Hide file tree
Showing 23 changed files with 63 additions and 24 deletions.
3 changes: 2 additions & 1 deletion pipelines/datasets/br_ans_beneficiario/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
with Flow(
name="br_ans_beneficiario.informacao_consolidada",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as datasets_br_ans_beneficiario_flow:
dataset_id = Parameter("dataset_id", default="br_ans_beneficiario", required=False)
Expand Down Expand Up @@ -114,6 +114,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
1 change: 1 addition & 0 deletions pipelines/datasets/br_b3_cotacoes/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
1 change: 1 addition & 0 deletions pipelines/datasets/br_bcb_agencia/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
2 changes: 2 additions & 0 deletions pipelines/datasets/br_bcb_estban/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -230,6 +231,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
1 change: 1 addition & 0 deletions pipelines/datasets/br_bd_indicadores/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
9 changes: 6 additions & 3 deletions pipelines/datasets/br_cgu_beneficios_cidadao/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
with Flow(
name="br_cgu_beneficios_cidadao.novo_bolsa_familia",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as datasets_br_cgu_bolsa_familia_flow:
dataset_id = Parameter(
Expand Down Expand Up @@ -125,6 +125,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -170,7 +171,7 @@
with Flow(
name="br_cgu_beneficios_cidadao.garantia_safra",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as datasets_br_cgu_garantia_safra_flow:
dataset_id = Parameter(
Expand Down Expand Up @@ -256,6 +257,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -297,7 +299,7 @@
with Flow(
name="br_cgu_beneficios_cidadao.bpc",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as datasets_br_cgu_bpc_flow:
dataset_id = Parameter(
Expand Down Expand Up @@ -381,6 +383,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
2 changes: 1 addition & 1 deletion pipelines/datasets/br_cnj_improbidade_administrativa/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
},
labels=current_flow_labels,
run_name=r"Materialize {dataset_id}.{table_id}",
upstream_tasks=[current_flow_labels],
upstream_tasks=[wait_upload_table],
)

wait_for_materialization = wait_for_flow_run(
Expand Down
18 changes: 12 additions & 6 deletions pipelines/datasets/br_cvm_fi/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
with Flow(
name="br_cvm_fi_documentos_informe_diario",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_informe_diario:
# Parameters
Expand Down Expand Up @@ -111,6 +111,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -148,7 +149,7 @@
with Flow(
name="br_cvm_fi_documentos_carteiras_fundos_investimento",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_carteiras_fundos_investimento:
# Parameters
Expand Down Expand Up @@ -211,6 +212,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -250,7 +252,7 @@
with Flow(
name="br_cvm_fi_documentos_extratos_informacoes",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_extratos_informacoes:
# Parameters
Expand Down Expand Up @@ -317,6 +319,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -355,7 +358,7 @@
with Flow(
name="br_cvm_fi_documentos_perfil_mensal",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_perfil_mensal:
# Parameters
Expand Down Expand Up @@ -414,6 +417,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -450,7 +454,7 @@
with Flow(
name="br_cvm_fi_documentos_informacao_cadastral",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_informacao_cadastral:
# Parameters
Expand Down Expand Up @@ -508,6 +512,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -546,7 +551,7 @@
with Flow(
name="br_cvm_fi_documentos_balancete",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_cvm_fi_documentos_balancete:
# Parameters
Expand Down Expand Up @@ -606,6 +611,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
2 changes: 2 additions & 0 deletions pipelines/datasets/br_denatran_frota/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks=[wait_upload_table],
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -225,6 +226,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks=[wait_upload_table],
)

wait_for_materialization = wait_for_flow_run(
Expand Down
3 changes: 2 additions & 1 deletion pipelines/datasets/br_inmet_bdmep/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
# from pipelines.datasets.br_ibge_pnadc.schedules import every_quarter

# pylint: disable=C0103
with Flow(name="br_inmet_bdmep", code_owners=["arthurfg"]) as br_inmet:
with Flow(name="br_inmet_bdmep", code_owners=["equipe_pipelines"]) as br_inmet:
# Parameters
dataset_id = Parameter("dataset_id", default="br_inmet_bdmep", required=False)
table_id = Parameter("table_id", default="microdados", required=False)
Expand Down Expand Up @@ -78,6 +78,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
8 changes: 4 additions & 4 deletions pipelines/datasets/br_me_cnpj/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
with Flow(
name="br_me_cnpj.empresas",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_me_cnpj_empresas:
dataset_id = Parameter("dataset_id", default="br_me_cnpj", required=False)
Expand Down Expand Up @@ -129,7 +129,7 @@
with Flow(
name="br_me_cnpj.socios",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_me_cnpj_socios:
dataset_id = Parameter("dataset_id", default="br_me_cnpj", required=False)
Expand Down Expand Up @@ -222,7 +222,7 @@
with Flow(
name="br_me_cnpj.estabelecimentos",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
executor=LocalDaskExecutor()
) as br_me_cnpj_estabelecimentos:
Expand Down Expand Up @@ -361,7 +361,7 @@
with Flow(
name="br_me_cnpj.simples",
code_owners=[
"arthurfg",
"equipe_pipelines",
],
) as br_me_cnpj_simples:
dataset_id = Parameter("dataset_id", default="br_me_cnpj", required=True)
Expand Down
4 changes: 4 additions & 0 deletions pipelines/datasets/br_me_comex_stat/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -222,6 +223,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -332,6 +334,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -438,6 +441,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
1 change: 1 addition & 0 deletions pipelines/datasets/br_mg_belohorizonte_smfa_iptu/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
6 changes: 6 additions & 0 deletions pipelines/datasets/br_ons_avaliacao_operacao/flows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -198,6 +199,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -299,6 +301,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -401,6 +404,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -504,6 +508,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down Expand Up @@ -605,6 +610,7 @@
},
labels=current_flow_labels,
run_name=f"Materialize {dataset_id}.{table_id}",
upstream_tasks = [wait_upload_table]
)

wait_for_materialization = wait_for_flow_run(
Expand Down
Loading

0 comments on commit 08c37bb

Please sign in to comment.