Plev compile fix

postprocessing/plevel_interpolation/compile_plev_interpolation.sh

@@ -5,6 +5,9 @@ cd ./exec
 
 source $GFDL_BASE/src/extra/env/$GFDL_ENV
 
-../bin/mkmf -p plev.x -t ../bin/mkmf.template.ia64 -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include
+compiler=${GFDL_MKMF_TEMPLATE:-ia64}
+template=mkmf.template.${compiler}
+
+../bin/mkmf -p plev.x -t $GFDL_BASE/src/extra/python/isca/templates/$template -c "-Duse_netCDF" -a ../src ../src/path_names ../src/shared/mpp/include ../src/shared/include
 
 make -f Makefile

postprocessing/plevel_interpolation/scripts/run_plevel_simple.py

@@ -0,0 +1,51 @@
+from netCDF4 import Dataset  
+from plevel_fn import plevel_call, daily_average, join_files, two_daily_average, monthly_average
+import sys
+import os
+import time
+import pdb
+import subprocess
+
+start_time=time.time()
+base_dir=os.environ['GFDL_DATA']
+exp_name_list = ['exp_name_here']
+avg_or_daily_list=['daily']
+start_file=1
+end_file=12
+nfiles=(end_file-start_file)+1
+
+level_set='standard' #Default is the standard levels used previously. ssw_diagnostics are the ones blanca requested for MiMa validation
+mask_below_surface_set=' ' #Default is to mask values that lie below the surface pressure when interpolated. For some applications, you want to have values interpolated below ground, i.e. as if the ground wasn't there. To use this option, this value should be set to '-x '. 
+
+try:
+    out_dir
+except:
+    out_dir = base_dir
+
+plevs={}
+var_names={}
+
+if level_set=='standard':
+
+    plevs['monthly']=' -p "1000 10000 25000 50000 85000 92500"'
+    plevs['6hourly']=' -p "1000 10000 25000 50000 85000 92500"'
+    plevs['daily']  =' -p "1000 10000 25000 50000 85000 92500"'
+
+    var_names['monthly']='-a slp height'
+    var_names['6hourly']='-a slp height'
+    var_names['daily']='-a slp height'
+    file_suffix='_interp'
+
+for exp_name in exp_name_list:
+    for n in range(nfiles):
+        for avg_or_daily in avg_or_daily_list:
+            print(n+start_file)
+
+            nc_file_in = base_dir+'/'+exp_name+'/run%04d'%(n+start_file)+'/atmos_'+avg_or_daily+'.nc'
+            nc_file_out = out_dir+'/'+exp_name+'/run%04d'%(n+start_file)+'/atmos_'+avg_or_daily+file_suffix+'.nc'
+
+            if not os.path.isfile(nc_file_out):
+                plevel_call(nc_file_in,nc_file_out, var_names = var_names[avg_or_daily], p_levels = plevs[avg_or_daily], mask_below_surface_option=mask_below_surface_set)
+
+print('execution time', time.time()-start_time)
+

src/extra/python/scripts/remove_certain_restart_and_data_files.py

@@ -1,6 +1,7 @@
 import sh
 import os
 import pdb
+from glob import glob
 
 P = os.path.join
 
@@ -12,46 +13,58 @@ def __init__(self, basedir, workdir, datadir, exp_name):
         self.expname = exp_name
 
 
-def create_exp_object(exp_name):
+def create_exp_object(exp_name, data_directory=None):
+
+    if data_directory is None:
+        datadir = os.environ['GFDL_DATA']
+    else:
+        datadir = data_directory
 
-    basedir = os.environ['GFDL_BASE']
     workdir = os.environ['GFDL_WORK']
-    datadir = os.environ['GFDL_DATA']
+    basedir = os.environ['GFDL_BASE']
     expname = '/'+exp_name+'/'
 
     exp_object = temporary_exp_object(basedir, workdir, datadir, exp_name)
 
     return exp_object
 
 
-def keep_only_certain_restart_files(exp_object, max_num_files, interval=12):
+# def keep_only_certain_restart_files(exp_object, max_num_files, interval=12):
 
-    #        sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass.
+#     #        sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass.
 
-            #First defines a list of ALL the restart file numbers
-        files_to_remove=list(range(0,max_num_files))
+#             #First defines a list of ALL the restart file numbers
+#         files_to_remove=list(range(0,max_num_files))
 
-            #Then defines a list of the ones we want to KEEP
-        files_to_keep  =list(range(0,max_num_files,interval)) 
+#             #Then defines a list of the ones we want to KEEP
+#         files_to_keep  =list(range(0,max_num_files,interval)) 
 
-            #Then we remove the files we want to keep from the list of all files, giving a list of those we wish to remove
-        for x in files_to_keep:
-               files_to_remove.remove(x) 
+#             #Then we remove the files we want to keep from the list of all files, giving a list of those we wish to remove
+#         for x in files_to_keep:
+#                files_to_remove.remove(x) 
 
-            #Then we remove them.
-        for entry in files_to_remove:
-            try:
-                sh.rm(P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio'))
-#                 print P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio')
+#             #Then we remove them.
+#         for entry in files_to_remove:
+#             try:
+#                 sh.rm(P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio'))
+# #                 print P(exp_object.workdir,exp_object.expname,'restarts','res_'+str(entry)+'.cpio')
 
-            except sh.ErrorReturnCode_1:
-                pass
-#                 print 'Tried to remove some restart files, but number '+str(entry)+' does not exist'
+#             except sh.ErrorReturnCode_1:
+#                 pass
+# #                 print 'Tried to remove some restart files, but number '+str(entry)+' does not exist'
 
-def keep_only_certain_restart_files_data_dir(exp_object, max_num_files, interval=12):
+def keep_only_certain_restart_files_data_dir(exp_object, max_num_files=None, interval=12):
 
     #        sh.ls(sh.glob(P(self.workdir,'restarts','res_*.cpio'))) #TODO get max_num_files calculated in line, rather than a variable to pass.
 
+        if max_num_files is None:
+            month_list = glob(P(exp_object.datadir,exp_object.expname, 'restarts')+'/res*.tar.gz')
+            if len(month_list)==0:
+                return
+            else:
+                final_month = month_list[-1].split('/res')
+            max_num_files = int(final_month[-1].split('.tar.gz')[0])
+
             #First defines a list of ALL the restart file numbers
         files_to_remove=list(range(0,max_num_files))
 
@@ -62,13 +75,27 @@ def keep_only_certain_restart_files_data_dir(exp_object, max_num_files, interval
         for x in files_to_keep:
                files_to_remove.remove(x) 
 
+        first_to_be_removed = True
+        number_removed = 0
+        number_not_removed = 0
             #Then we remove them.
-        for entry in files_to_remove:
+        for entry in files_to_remove[1:-1]:
             try:
-                sh.rm(P(exp_object.datadir,exp_object.expname,'run%03d' % entry,'INPUT','res'))
-#                 print 'would be removing ' + P(exp_object.datadir,exp_object.expname,'run'+str(entry),'INPUT','res')
+                file_to_remove = P(exp_object.datadir,exp_object.expname, 'restarts', 'res%04d.tar.gz' % entry)
+                if os.path.isfile(file_to_remove) and first_to_be_removed:
+                    first_to_be_removed=False
+                    number_not_removed+=1
+                    # print('would have removed '+file_to_remove+' but wanted to make sure not to delete the first restart')
+                else:
+                    sh.rm(file_to_remove)
+                    number_removed+=1
+                    # print('have removed ' + file_to_remove)
             except sh.ErrorReturnCode_1:
+                number_not_removed+=1                
+                # print('could not remove ' + file_to_remove)
                 pass
+        print(P(exp_object.datadir,exp_object.expname), 'number removed '+str(number_removed), 'number not removed '+str(number_not_removed))
+
 #                 print 'Tried to remove some restart files, but number '+str(entry)+' does not exist'                
 
 def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=None, file_name = 'atmos_daily.nc'):
@@ -88,8 +115,8 @@ def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=No
             #Then we remove them.
         for entry in files_to_remove:           
             try:
-                sh.rm(P(exp_object.datadir,exp_object.expname,'run%03d' % entry,file_name))
-                print(('Removed '+P(exp_object.datadir,exp_object.expname,'run%03d' % entry,file_name)))        
+                sh.rm(P(exp_object.datadir,exp_object.expname,'run%04d' % entry,file_name))
+                print(('Removed '+P(exp_object.datadir,exp_object.expname,'run%04d' % entry,file_name)))        
             except sh.ErrorReturnCode_1:
                 pass
 #                 print 'Tried to remove some atmos_daily files, but number '+str(entry)+' does not exist'
@@ -98,36 +125,18 @@ def keep_only_certain_daily_data_uninterp(exp_object, max_num_files, interval=No
 
 if __name__=="__main__":
 
-    max_num_files_input = 325
+    max_num_files_input = None
 
-#     exp_name_list=['simple_continents_post_princeton_qflux_anoms_'+str(x) for x in range(31,32)]
-
-#     exp_name_list=['aquaplanet_qflux_anoms_'+str(x) for x in [12,18,23,32,8]]
-
-#     exp_name_list = ['simple_continents_post_princeton_qflux_control_1','simple_continents_post_princeton_fixed_sst_1', 'simple_continents_post_princeton_qflux_control_nod_1', 'simple_continents_post_princeton_qflux_control_scf_1']
-#     
-#     exp_name_list = ['annual_mean_ice_princeton_qflux_control_matrix_qflux_2017_code_1', 'annual_mean_ice_post_princeton_fixed_sst_1', 'annual_mean_ice_princeton_fixed_sst_1'] 
-# 
-#     exp_name_list.extend(['annual_mean_ice_post_princeton_fixed_sst_el_nino_1'])
-
-#     exp_name_list = ['simple_continents_post_princeton_qflux_control_1']
-
-#     exp_name_list = ['annual_mean_ice_princeton_qflux_control_1']#, 'annual_mean_ice_post_princeton_qflux_control_1']
-
-#     exp_name_list = ['annual_mean_ice_post_princeton_fixed_sst_TEST_1', 'annual_mean_ice_princeton_qflux_control_matrix_qflux_1']
-
-#     exp_name_list.extend(['simple_continents_post_princeton_fixed_sst_1'])
-
-#     exp_name_list = ['giant_drag_exp_chai_values_without_dc_bug_latest_1']
-#     exp_name_list = ['aquaplanet_qflux_control_1']
+    # exp_name_list = ['']
+    exp_name_list = glob('/disca/share/sit204/data_from_isca_cpu/*/')
 
-    exp_name_list = ['giant_drag_exp_chai_values_with_dc_bug_latest_start_to_finish_1', 'giant_drag_exp_chai_values_without_dc_bug_latest_start_to_finish_1']
 
     for exp_name_input in exp_name_list:    
-        temp_obj = create_exp_object(exp_name_input)
-        keep_only_certain_restart_files(temp_obj, max_num_files_input)
+        print('Percentage progress through list:'+str(exp_name_list.index(exp_name_input)/len(exp_name_list)))
+        temp_obj = create_exp_object(exp_name_input, data_directory='/disca/share/sit204/data_from_isca_cpu/')
+        # keep_only_certain_restart_files(temp_obj, max_num_files_input)
         keep_only_certain_restart_files_data_dir(temp_obj, max_num_files_input)
-        keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input, file_name = 'fms_moist.x')
+        # keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input, file_name = 'fms_moist.x')
 #         keep_only_certain_daily_data_uninterp(temp_obj, max_num_files_input)