From 27beb548873d14e155d26255de77e307081b8579 Mon Sep 17 00:00:00 2001 From: taimur-10xe <92528065+taimur-10xe@users.noreply.github.com> Date: Sat, 18 Mar 2023 12:57:45 +0500 Subject: [PATCH] DG, CFA, CSC enabled - PR Request (#6) * automate_execution currently works on config_automate file to generate RTL compatible results. * added in_single_folder flag to set the structure of the out_for_RTL folder. If True, all input files are saved in a single folder. * - automate_execution.py works on configs.yml now instead of a separate config_automate.yml. - added is_save flag in configs.yml * - verified results by matching .npy files, they match. - exception added if ValueError is encountered while loading np arrays. * - foldername(line 63) now specified by datetime obj - added compare_output file (Inprogress) - made chnages to handle new foldername in rename_to_RTL in utils.py * output of the default modules can now by saved using is_save flag in automate_execution script. * To test multiple modules, type of DUT changed from str to list. * added input_ext variable to specify raw file extension. --------- Co-authored-by: Maria_Nadeem-10x --- automate_execution.py | 50 ++++++++++++++++++++++++++++--------------- compare_output.py | 20 ++++++++--------- util/utils.py | 12 +++++------ 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/automate_execution.py b/automate_execution.py index deb2e6b..d88e1f0 100644 --- a/automate_execution.py +++ b/automate_execution.py @@ -14,7 +14,10 @@ # DUT options = ['crop', 'dead_pixel_correction', 'hdr_stitching', 'black_level_correction', 'OECF', 'digital_gain', 'lens_shading_correction', # 'bayer_noise_reduction', 'white_balance', 'demosaic', 'auto_white_balance', 'color_correction_matrix', 'gamma_correction', # 'auto_exposure', 'color_space_conversion', 'ldci', 'sharpen', '2d_noise_reduction', 'scale', 'yuv_conversion_format'] -DUT = "black_level_correction" + + +DUT = ["black_level_correction"] + # Modules: CROP = {"isEnable": False, @@ -28,10 +31,15 @@ HDR = {"isEnable": False} BLC = {"isEnable": True, - "isLinear": False} + "isLinear": False, + "r_offset": 100, + "gr_offset": 110, + "gb_offset": 120, + "b_offset": 130} OECF = {"isEnable": False} -DG = {"is_save": True} +DG = {"is_save": False} + LSC = {"isEnable": False} BNR = {"isEnable": False, @@ -44,8 +52,10 @@ CCM = {"isEnable": False} GC = {"isEnable": False} AE = {"isEnable": False} -CSC = {"is_save": True} -LDCI = {"isEnable": True} + +CSC = {"is_save": False} +LDCI = {"isEnable": False} + SHARPEN ={"isEnable": False} NR2D = {"isEnable": False} @@ -63,6 +73,9 @@ # define folder name to save outputs folder_name = datetime.now().strftime("%Y%m%d_%H%M%S") +input_ext = ".bin" + + # Set this flag to True if all input files need to be saved in a single folder in_single_folder = True @@ -84,27 +97,28 @@ config = yaml.safe_load(file) module_tags = list(config.keys())[2:] -default_modules = ["pre_gamma", "tone_mapping", "jpeg_conversion"] -[module_tags.remove(module) for module in default_modules] +remove = ["pre_gamma", "tone_mapping", "jpeg_conversion"] +[module_tags.remove(module) for module in remove] # ensure that the modules are in same order as they are in the config file new_params = [CROP, DPC, HDR, BLC, OECF, DG, LSC, BNR, WB, DEM, AWB, CCM, GC, AE, CSC, LDCI, SHARPEN, NR2D, SCALE, YUV] +default_modules = ["digital_gain", "demosaic", "color_space_conversion"] + for idx, module in enumerate(module_tags): # save the input and output arrays of module under test with is_save flag - try: - # is_save = True if module==DUT or module_tags[idx+1]==DUT else False - - if module==DUT or module_tags[idx+1]==DUT: + try: + if module in DUT or module_tags[idx+1] in DUT: is_save = True - elif module in ["digital_gain", "demosaic", "color_space_conversion"]: + elif module in default_modules: + is_save = new_params[idx]["is_save"] else: is_save = False - if module==DUT: + if module in DUT and module not in default_modules: assert new_params[idx]["isEnable"]==True, "DUT not enabled in this script." utils.update_config(config, module,new_params[idx].keys(), new_params[idx].values(), is_save) except IndexError: @@ -144,16 +158,18 @@ # rename output files of the previous module to DUT as "In_" to identify these files # as input to RTL files path = f"./out_for_RTL/{folder_name}/" -if DUT=="crop": + +if "crop" in DUT: utils.rename_for_RTL(path, "Inpipeline_crop_") else: - input_module = module_tags[module_tags.index(DUT)-1] + input_module = module_tags[module_tags.index(DUT[0])-1] utils.rename_for_RTL(path, "Out_" + input_module + "_") # restructure the directory -utils.restructure_dir(path, in_single_folder, DUT) +utils.restructure_dir(path, in_single_folder, DUT[-1]) + # convert the saved numpy arrays to bin files as required by the RTL -utils.get_RTL_input(path, in_single_folder) +utils.get_RTL_input(path, in_single_folder, input_ext) diff --git a/compare_output.py b/compare_output.py index 9f4827c..ca45e7e 100644 --- a/compare_output.py +++ b/compare_output.py @@ -3,9 +3,9 @@ import numpy as np # Define paths -path_to_RTL_results = "./out_for_RTL/BLC/" -path_to_softISP_results = "./out_for_RTL/Results/Output/" -remove_border = 2 +path_to_RTL_results = "./out_for_RTL/RTL_BLC/" +path_to_softISP_results = "./out_for_RTL/infISP_BLC/Output/" +remove_border = 0 RTL_files = [file for file in os.listdir(path_to_RTL_results) if file[-4:]==".bin"] softISP_files = os.listdir(path_to_softISP_results) @@ -13,23 +13,23 @@ # sort the lists so RTL_files.sort() softISP_files.sort() -# print(softISP_files) + + for rtl_file in RTL_files: - nametag = "_" + rtl_file.split("_")[1] +"_" + nametag = "_" + rtl_file.split("_")[1] # +"_" w, h = int(rtl_file.split("x")[0][-4:]), int(rtl_file.split("x")[1][0:4]) - print(nametag) + # load files corresponding to the nametag from both RTL and softISP results infisp_file = [f_name for f_name in softISP_files if nametag in f_name] rtl_path = path_to_RTL_results + rtl_file infisp_path = path_to_softISP_results + infisp_file[0] - print(rtl_path, infisp_path) # load the infisp output infisp_arr = np.load(infisp_path, allow_pickle=False) - infisp_patch = infisp_arr[0:10, 0:10] + # remove border if specified if remove_border: @@ -43,9 +43,7 @@ # for 1 channel output if len(infisp_arr.shape)==2: rtl_arr = arr[::3].reshape((h,w)) - print(np.min(rtl_arr), np.max(rtl_arr)) - print(len(np.unique(rtl_arr))) - rtl_patch = rtl_arr[0:10, 0:10] + # for 3 channel output else: diff --git a/util/utils.py b/util/utils.py index 2665737..be8503c 100644 --- a/util/utils.py +++ b/util/utils.py @@ -251,7 +251,7 @@ def restructure_dir(path, singlefolder, out_module_name): if "Out_" + out_module_name + "_" in file: move(path+file, str(out_dir)+"/"+file) -def convert_npytobin(path): +def convert_npytobin(path, raw_ext=".bin"): """This function takes a numpy array and saves each channel of the single file (.npy) as a binary file (.bin) in the given directory.""" @@ -266,17 +266,17 @@ def convert_npytobin(path): if len(np_arr.shape)==3: # out =[] for idx, ch in enumerate("RGB"): - filename = ch + "_" + in_filename + ".bin" + filename = ch + "_" + in_filename + raw_ext # out.append(np_arr[:, :, idx].copy()) with open (dir_path+filename, "wb") as raw_file: np_arr[:, :, idx].tofile(raw_file) else: assert len(np_arr.shape)==2, "Input array must be a 2D array." - with open (dir_path+in_filename + ".bin", "wb") as raw_file: + with open (dir_path+in_filename + raw_ext, "wb") as raw_file: np_arr.tofile(raw_file) return np_arr -def get_RTL_input(path, singlefolder): +def get_RTL_input(path, singlefolder, input_ext): # for Input files conver npy to raw dirs = [dir for dir in os.listdir(path+"Input/")] @@ -286,11 +286,11 @@ def get_RTL_input(path, singlefolder): files = os.listdir(arr_path) assert len(files)==1, "Invalid path!" f_path = arr_path+files[0] - convert_npytobin(f_path) + convert_npytobin(f_path, input_ext) else: for file in dirs: f_path = path + "Input/" + file - convert_npytobin(f_path) + convert_npytobin(f_path, input_ext) # for output files change sequence of channels (h,w,ch)-->(ch,h,w) files = os.listdir(path+"Output/")