Merge branch 'main' into staging/cgu-licitacao-contrato

basedosdados · Dec 3, 2024 · 0abdbd0 · 0abdbd0
2 parents 7147b8d + 8a49317
commit 0abdbd0
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/pipelines/datasets/br_ms_sinan/flows.py b/pipelines/datasets/br_ms_sinan/flows.py
@@ -1,9 +1,7 @@
 # -*- coding: utf-8 -*-
 from copy import deepcopy
-
 from prefect.run_configs import KubernetesRun
 from prefect.storage import GCS
-
 from pipelines.utils.crawler_datasus.flows import flow_sinan
 from pipelines.constants import constants
 from pipelines.datasets.br_ms_sinan.schedules import (

diff --git a/pipelines/utils/crawler_datasus/flows.py b/pipelines/utils/crawler_datasus/flows.py
@@ -423,7 +423,6 @@
             file_list=dbc_files,
             dataset_id=dataset_id,
             table_id=table_id,
-            chunk_size = 150000,
             upstream_tasks=[dbf_files, dbc_files],
         )
 

diff --git a/pipelines/utils/crawler_datasus/utils.py b/pipelines/utils/crawler_datasus/utils.py
@@ -2,7 +2,7 @@
 """
 General purpose functions for the br_ms_cnes project
 """
-
+import gc
 import asyncio
 from datetime import datetime
 from ftplib import FTP
@@ -88,6 +88,12 @@ def  dbf_to_parquet(dbf: str, table_id: str, counter: int, chunk_size:int) -> st
 
                 df.to_parquet(parquet_filepath, index=None, compression='gzip')
 
+                del df
+
+                gc.collect()
+
+
+
 
     except struct.error as err:
         #unlink .partquer extension and remove dbf file