Merge branch 'dev' into postmodeling_revision_2023

src/triage/component/architect/builders.py

@@ -443,8 +443,6 @@ def feature_load_queries(self, feature_dictionary, entity_date_table_name):
                     table=entity_date_table_name,
                 ),
                 right_column_selections=[', "{0}"'.format(fn) for fn in feature_names],
-                include_index=True
-                #include_index=True if num==0 else False,
             ))
         return queries
 
@@ -589,10 +587,12 @@ def remove_unnecessary_files(self, filenames, path_, matrix_uuid):
         # deleting features and label csvs
         for filename_ in filenames:
            cmd_line = 'rm ' + filename_ 
+           logger.debug(f"removing files with command {cmd_line}")
            subprocess.run(cmd_line, shell=True)
 
         # deleting the merged csv
         cmd_line = 'rm ' + path_ + "/" + matrix_uuid + '.csv'
+        logger.debug(f"removing stitched csv with command {cmd_line}")
         subprocess.run(cmd_line, shell=True)
 
 

src/triage/component/architect/utils.py

@@ -116,9 +116,15 @@ def create_entity_date_df(
 
 
 def change_datetimes_on_metadata(metadata):
-    variables = ['end_time', 'feature_start_time', 'first_as_of_time', 'last_of_time', 'matrix_info_end_time']
-    for variable in variables:
-        metadata[variable] = str(metadata[variable])
+    metadata_keys = list(metadata.keys())
+
+    for element in metadata_keys: 
+        if element.endswith("_time"): 
+            metadata[element] = str(metadata[element])
+
+    #variables = ['end_time', 'feature_start_time', 'first_as_of_time', 'last_of_time', 'matrix_info_end_time']
+    #for variable in variables:
+    #    metadata[variable] = str(metadata[variable])
 
     return metadata
 
@@ -311,15 +317,17 @@ def remove_entity_id_and_knowledge_dates(filenames, matrix_uuid):
 def generate_list_of_files_to_remove(filenames, matrix_uuid):
     """Generate the list of all files that need to be removed"""
     # adding _sorted
-    rm_files = filenames 
+    rm_files = []
 
     for element in filenames:
+        rm_files.append(element)
         if (element.split("/")[-1].startswith(matrix_uuid)):
             prefix = element.split(".")[0]
             # adding sorted files 
             rm_files.append(prefix + "_sorted.csv")
             # adding fixed files
             rm_files.append(prefix + "_fixed.csv")
 
+    logging.debug(f"Files to be removed {rm_files}")
     return rm_files