From 15d3c01305330b3039d3de3095a2f699c134d6b1 Mon Sep 17 00:00:00 2001 From: Liliana Millan Date: Thu, 1 Feb 2024 17:11:41 +0000 Subject: [PATCH] save csv.gz in s3 --- src/triage/component/architect/builders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/triage/component/architect/builders.py b/src/triage/component/architect/builders.py index 34a1e9c5f..bd75e5d7d 100644 --- a/src/triage/component/architect/builders.py +++ b/src/triage/component/architect/builders.py @@ -308,7 +308,7 @@ def build_matrix( ) feature_queries = self.feature_load_queries(feature_dictionary, entity_date_table_name) - logger.spam(f"feature queries, number of queries: {len(feature_queries)}") + logger.debug(f"feature queries, number of queries: {len(feature_queries)}") label_query = self.label_load_query( label_name, @@ -322,7 +322,7 @@ def build_matrix( matrix_store.metadata = matrix_metadata #labels = output.pop(matrix_store.label_column_name) matrix_store.matrix_label_tuple = output, labels - #matrix_store.save() + matrix_store.save() logger.info(f"Saving matrix metadata (yaml) for matrix {matrix_uuid}") matrix_store.save_matrix_metadata() @@ -578,7 +578,7 @@ def stitch_csvs(self, features_queries, label_query, matrix_store, matrix_uuid): logger.debug(f"Time converting from polars to pandas (sec): {(end-start)/60}") df.set_index(["entity_id", "as_of_date"], inplace=True) logger.debug(f"df data types: {df.dtypes}") - logger.spam(f"Pandas DF memory usage: {df.memory_usage(deep=True).sum()/1000000} MB") + logger.debug(f"Pandas DF memory usage: {df.memory_usage(deep=True).sum()/1000000} MB") logger.debug(f"Generating gzip from full matrix csv") self.generate_gzip(path_, matrix_uuid)