17 merge functionality of electron classifier model and fake tracks model

DeepSets/test/train.py

@@ -4,90 +4,90 @@
 sys.path.append("/home/ryan/scratch0/CMSSW_12_4_11_patch3/src/")
 print("PATH: ", sys.path)
 import tensorflow as tf
-#import cmsml
-from sklearn.model_selection import KFold
-
-from DisappTrksML.DeepSets.python.ElectronModel import *
-print("Imported")
-# from DisappTrksML.DeepSets.MuonModel import *
-from DisappTrksML.DeepSets.python.generator import *
-from DisappTrksML.DeepSets.python.utilities import *
+from datetime import datetime
+import numpy as np
+import sys
+from DisappTrksML.DeepSets.ElectronModel import ElectronModel
+from DisappTrksML.DeepSets.generator import BalancedGenerator, Generator
+#from DisappTrksML.DeepSets.utilities import *
 
 if False:
-	# limit CPU usage
-	config = tf.compat.v1.ConfigProto(inter_op_parallelism_threads = 4,   
-									intra_op_parallelism_threads = 4,
-									allow_soft_placement = True,
-									device_count={'CPU': 4})
-	tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=config))
+        # limit CPU usage
+        config = tf.compat.v1.ConfigProto(inter_op_parallelism_threads = 4,   
+                                                                        intra_op_parallelism_threads = 4,
+                                                                        allow_soft_placement = True,
+                                                                        device_count={'CPU': 4})
+        tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=config))
 
 #######
-
-backup_suffix = "backup"
+os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+backup_suffix = datetime.now().strftime('%Y-%m-%d_%H.%M.%S')
 outdir = "train_"+backup_suffix+"/"
-if(len(sys.argv)>1):
-	input_params = np.load("params.npy",allow_pickle=True)[int(sys.argv[1])]
-	print(input_params)
-	outdir = str(input_params[-1])
-
 print("output directory", outdir)
 
-info_indices = [4, 	# nPV
-				8, 	# eta
-				9, 	# phi
-				12	# nValidPixelHits
-				]
+info_indices = [4,      # nPV
+                                8,      # eta
+                                9,      # phi
+                                12      # nValidPixelHits
+                                ]
 model_params = {
-	'eta_range':0.25,
-	'phi_range':0.25,
-	'phi_layers':[64,64,256],
-	'f_layers':[64,64,64],
-	'max_hits' : 100,
-	'track_info_indices' : info_indices
-}	
+        'eta_range':0.25,
+        'phi_range':0.25,
+        'phi_layers':[64,64,256],
+        'f_layers':[64,64,64],
+        'max_hits' : 100,
+        'track_info_indices' : info_indices
+}       
 val_generator_params = {
-	'input_dir' : '/store/user/mcarrigan/deepSets/electronModel/',
-	'batch_size' : 256,
-	'max_hits' : 100,
-	'info_indices' : info_indices
+        'input_dir' : '/store/user/rsantos/2022/combined_DYJet/training/',
+        'batch_size' : 256,
+        'max_hits' : 100,
+        'info_indices' : info_indices
 }
 train_generator_params = val_generator_params.copy()
 train_generator_params.update({
-	'shuffle': True,
-	'batch_ratio': 0.5
+        'shuffle': True,
+        'batch_ratio': 0.5
 })
 train_params = {
-	'epochs': 1,
-	'outdir':outdir,
-	'patience_count':5
+        'epochs': 1,
+        'outdir':outdir,
+        'patience_count':20
 }
 
-if(not os.path.isdir(outdir)): os.mkdir(outdir)
+if(len(sys.argv)>1):
+        input_params = np.load("params.npy",allow_pickle=True)[int(sys.argv[1])]
+        print(input_params)
+        train_params['outdir'] = str(input_params[-1])
+        model_params['phi_layers'] = input_params[0]
+        model_params['f_layers'] = input_params[1]
+
+
+if(not os.path.isdir(train_params["outdir"])): os.mkdir(train_params["outdir"])
 
 arch = ElectronModel(**model_params)
-arch.buildModel()
+arch.build_model()
 
-inputFiles = glob.glob(train_generator_params['input_dir']+'images_*10.root.npz')
+inputFiles = glob.glob(train_generator_params['input_dir']+'*.root.npz')
 inputIndices = np.array([f.split('images_')[-1][:-9] for f in inputFiles])
 nFiles = len(inputIndices)
 print(('Found', nFiles, 'input files'))
 
 file_ids = {
-	'train'      : inputIndices[:10],
-	'validation' : inputIndices[10:]
+        'train'      : inputIndices[:int(nFiles*0.9)],
+        'validation' : inputIndices[int(nFiles*0.9):]
 }
 
 train_generator = BalancedGenerator(file_ids['train'], **train_generator_params)
 val_generator = Generator(file_ids['validation'], **val_generator_params)
 
-arch.fit_generator(train_generator=train_generator, 
-				   val_generator=val_generator, 	
-					**train_params)
+arch.train_model(data_directory= train_generator_params['input_dir'],
+                 epochs = 1,
+                 patience_count = 20,
+                 outdir = train_params['outdir'],
+                 val_generator_params = val_generator_params,
+                 train_generator_params = train_generator_params)
 
 #arch.saveGraph()
 
-arch.save_trainingHistory(train_params['outdir']+'trainingHistory.pkl')
-arch.plot_trainingHistory(train_params['outdir']+'trainingHistory.pkl',train_params['outdir']+'trainingHistory.png','loss')
-arch.save_weights(train_params['outdir']+'model_weights.h5')
-arch.save_model(train_params['outdir']+'model.h5')
-arch.save_metrics(train_params['outdir']+'trainingHistory.pkl',train_params['outdir']+"metrics.pkl", train_params)
+#arch.save_model(train_params['outdir']+'model.h5')

MachineLearning/README.md

@@ -0,0 +1,6 @@
+* Instructions to use NetworkController()
+  - Any network that you want to use with the NetworkController should subclass the NetworkBase class defined in networkController.py
+  - All functions defined within NetworkBase should be reimplemented inside your class with the same signatures (ie. same input and output)
+  - After this all functionality of NetworkController() should be available to your model.
+  - When writing tools for both the DeepSets model and the FakeTracks model, only assume that the methods defined in NetworkController exist. If you find you need another method, add this to NetworkController() and make sure that these methods are added to both models. 
+The code is well documented, so check networkController for further details on what is available for use and how to use it. An example for using the NetworkController is given in example.py. 

MachineLearning/condor_generate_example.py

@@ -0,0 +1,32 @@
+"""
+An example of how you would use
+generate_condor_submission to create a condor
+submission script
+"""
+import sys
+import os
+import pickle
+import logging
+sys.path.append("/data/users/mcarrigan/home_link/.local/bin/")
+#sys.path.append('/home/ryan/Documents/Research/')
+import optuna
+from networkController import NetworkController
+
+logging.basicConfig(level=logging.DEBUG)
+
+params = [{"trainable_params":{"phi_layers" : ["layers", 1, 3, 8, 16]}},
+          {"trainable_params":{"f_layers" : ["layers", 1, 3, 8, 16]}}]
+
+with open("params.pkl", "wb") as pickle_file:
+    pickle.dump(params, pickle_file)
+
+NetworkController.generate_condor_submission(
+    argument="$(PROCESS)",
+    number_of_jobs=len(params),
+    use_gpu=True, log_dir=".",
+    input_files=["run_wrapper_gpu.sh", "runNN.py",
+                 "ElectronModel.py", "networkController.py",
+                 "generator.py", "singularity_wrapper.sh"
+                 ])
+
+os.system("condor_submit run.sub")

MachineLearning/condor_script_generator.py

@@ -0,0 +1,37 @@
+class HTCondorScriptGenerator:
+    def __init__(self, executable, output_file="out_$(PROCESS).txt", error_file="error_$(PROCESS).txt", log_file="log_$(PROCESS).log", log_dir="."):
+        self.script_lines = []
+        self.script_lines.append("Executable = {}".format(executable))
+        self.script_lines.append("Output = {}/{}".format(log_dir, output_file))
+        self.script_lines.append("Error = {}/{}".format(log_dir, error_file))
+        self.script_lines.append("Log = {}/{}".format(log_dir,log_file))
+
+    def add_argument(self, argument):
+        self.script_lines.append("Arguments = {}".format(argument))
+
+    def add_requirements(self, requirements):
+        self.script_lines.append("Requirements = {}".format(requirements))
+
+    def add_queue(self, num_jobs=1):
+        self.script_lines.append("Queue {}".format(num_jobs))
+
+    def add_option(self, option:str, value:str):
+        self.script_lines.append("{} = {}".format(option, value))
+
+    def generate_script(self, file_path):
+        with open(file_path, "w") as file:
+            file.write("\n".join(self.script_lines))
+
+# Example usage:
+if __name__ == "__main__":
+    # Create an instance of HTCondorScriptGenerator
+    condor_generator = HTCondorScriptGenerator(executable="my_executable")
+
+    # Add arguments, requirements, and specify the number of jobs to queue
+    condor_generator.add_argument("arg1 arg2 arg3")
+    condor_generator.add_requirements('OpSysMajorVer == 7')
+    condor_generator.add_option("should_transfer_files", "yes")
+
+    condor_generator.add_queue(num_jobs=5)
+    # Generate the HTCondor script and save it to a file
+    condor_generator.generate_script("my_condor_script.submit")