Merge pull request #37 from NOAA-GFDL/9.configure-script-update

9.configure script update

fre/frepp/configUpdates/config2.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+
+import os
+import yaml
+import click
+from jsonschema import validate, ValidationError, SchemaError
+import json
+
+######VALIDATE#####
+package_dir = os.path.dirname(os.path.abspath(__file__))
+schema_path = os.path.join(package_dir, 'schema-pp.json')
+
+def validateYaml(file):
+  # Load the json schema: .load() (vs .loads()) reads and parses the json in one
+  with open(schema_path) as s:
+    schema = json.load(s)
+
+  # Validate yaml
+  # If the yaml is not valid, the schema validation will raise errors and exit
+  if validate(instance=file,schema=schema) == None:
+    print("YAML VALID")
+
+###################
+@click.command()
+@click.option("-y",
+              type=str,
+              help="YAML file to be used for parsing",
+              required=True)
+
+#maybe change name to parseYaml
+def yamlInfo(y):
+#file="pp.yaml"
+  with open(y,'r') as f:
+    y=yaml.safe_load(f)
+    yml = validateYaml(y)
+
+#PARSE YAML
+## Write the rose-suite-exp configuration
+  for key,value in y.items():
+    #print(key)
+
+    ## Add: check paths?
+    if key == "configuration_paths":
+      #print(value)
+      for filename,path in value.items():
+        if filename == "rose-suite":
+          rs = path
+          # Create rose-suite-exp config
+          with open(rs,'w') as f:
+            f.write('[template variables]\n')
+            f.write('## Information for requested postprocessing, info to pass to refineDiag/preanalysis scripts, info for epmt, and info to pass to analysis scripts \n')
+
+        ## STILL CREATE ROSE APPS FOR NOW (regrid and remap)
+        elif filename == "rose-remap":
+          remap_roseapp = path
+          with open(remap_roseapp,'w') as f:
+            f.write("[command]\n")
+            f.write("default=remap-pp-components\n")            
+        elif filename == "rose-regrid":
+          regrid_roseapp = path
+          with open(regrid_roseapp,'w') as f:
+            f.write("[command]\n")
+            f.write("default=regrid-xy\n")
+
+## Populate ROSE-SUITE-EXP config
+    if key == "rose-suite":
+      #print(value)
+      for suiteconfiginfo,dict in value.items():
+        #print(value)
+        for configkey,configvalue in dict.items():
+          #print(value)
+          if configvalue != None:
+            with open(rs,'a') as f:
+              k=configkey.upper()
+              if configvalue == True or configvalue == False:
+                f.write(f'{k}={configvalue}\n\n')
+              else:
+                if "pp_start" in configkey:
+                  f.write(f'{k}="{configvalue}OOTZ"\n\n')
+                elif "pp_stop" in configkey:
+                  f.write(f'{k}="{configvalue}OOTZ"\n\n')
+                else:
+                  f.write(f'{k}="{configvalue}"\n\n')
+
+##if value of type is in remap AND regrid grid value in remap rose app=regrid-xy
+    if key == "regrid-remap-info":
+      for components,compinfo in value.items():
+        for i in compinfo:
+          for compkey,compvalue in i.items():
+            # Create/write remap rose app
+            with open(remap_roseapp,'a') as f:
+              if compkey == "type": 
+                f.write(f"\n\n[{compvalue}]\n")
+
+                #if xyInterp doesnt exist, grid is native
+                if i.get("xyInterp") == None:
+                  f.write(f"grid=native\n")
+                #in xyInterp exists, component can be regridded
+                elif i.get("xyInterp") != None:
+                  f.write("grid=regrid-xy\n")                       
+
+              elif compkey == "sources":
+                f.write(f"{compkey}={compvalue} ")
+              elif compkey == "timeSeries":
+                #print(compvalue)
+                for i in compvalue:
+                  #print(i)  
+                  for key,value in i.items():
+                    if key == "source":
+                      f.write(f"{value} ")
+
+            # Create/write regrid rose app
+            with open(regrid_roseapp,'a') as f:
+              # if xyInterp is defined, that is a regridded component
+              if i.get("xyInterp") != None:
+                if compkey == "type":
+                  f.write(f"\n[{compvalue}]\n")
+                  if "atmos" in compvalue:
+                    f.write("inputRealm=atmos\n")
+                  elif "land" in compvalue:
+                    f.write("inputRealm=land\n")
+                  elif "river" in compvalue:
+                    f.write("inputRealm=land\n")
+                  elif "ocean" in compvalue:
+                    f.write("inputRealm=ocean\n")
+                  elif "aerosol" or "tracer" in compvalue:
+                    f.write("inputRealm=aerosol\n")
+                elif compkey == "sourceGrid": 
+                  f.write(f"inputGrid={compvalue}\n")
+                elif compkey == "interpMethod":
+                  f.write(f"{compkey}={compvalue}\n")
+                elif compkey == "sources":
+                  f.write(f"{compkey}={compvalue} ")
+                  if i["timeSeries"]:
+                    for elem in i['timeSeries']:
+                      for key,value in elem.items():
+                        if key == "source":
+                          f.write(f"{value} ")
+                  f.write("\n")
+
+# Use parseyaml function to parse created edits.yaml
+if __name__ == '__main__':
+    yamlInfo()

fre/frepp/configUpdates/pp.yaml

@@ -0,0 +1,225 @@
+## User defined edits
+configuration_paths: 
+  rose-suite: "/home/Dana.Singh/pp/ppyaml/fre-cli/fre/frepp/configUpdates/rose-suite-am5_c96L33_amip.conf" 
+  rose-remap: "/home/Dana.Singh/pp/ppyaml/fre-cli/fre/frepp/configUpdates/remap-rose-app.conf"
+  rose-regrid: "/home/Dana.Singh/pp/ppyaml/fre-cli/fre/frepp/configUpdates/regrid-rose-app.conf"
+
+analysisLen: &ANA_AMIP_LEN "10yr"
+chunk96: &PP_AMIP_CHUNK96 "1yr"
+analysisStart: &ANA_AMIP_START "1980"
+analysisStop: &ANA_AMIP_STOP "1988"
+defxyInterp: &defaultxyInterp "180,288"
+
+rose-suite:
+  refineDiag:
+    datastager_script: "$(includeDir)/refineDiag/refineDiag_data_stager_globalAve.csh"
+    atmoscmip6_script: "$(includeDir)/refineDiag/refineDiag_atmos_cmip6.csh"
+
+  settings:
+    history_segment: "P1Y"
+    pp_start: *ANA_AMIP_START
+    pp_stop: *ANA_AMIP_STOP
+    len: *ANA_AMIP_LEN
+    site: "ppan"
+    fre_analyis_home: #"/home/fms/local/opt/fre-analysis/test"
+    pp_chunk_a: "P5Y"
+    pp_chunk_b: "P10Y"
+    pp_grid_spec: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+
+  switches:
+    clean_work: True
+    do_mdtf: False
+    do_statics: True
+    do_timeavgs: True
+    do_refinediag: False
+    do_atmos_plevel_masking: True
+    do_preanalysis: False
+
+  directories:
+    history_dir: "/archive/c2b/am5/2022.01/c96L33_am5a0_cmip6Diag/gfdl.ncrc5-intel22-classic-prod-openmp/history"
+    pp_dir: "/archive/$USER/canopy/am5/c96L33_amip/pp"
+    ptmp: "/xtmp/$USER/ptmp"
+    refine_diag:
+    preanalysis:
+    history_refined: "/archive/$USER/canopy/am5/c96L33_amip/history_refineDiag"
+    analysis: #"/nbhome/$USER/canopy/analysis/am5_c96L33_amip"
+
+  metadata:
+    experiment: "am5_c96L33_amip"
+    platform: "gfdl.ncrc5-intel22-classic"
+    target: "prod-openmp"
+
+#regrid-xy and remap-pp-components information
+regrid-remap-info:
+  components:
+    - type: "atmos_cmip"
+      start: *ANA_AMIP_START
+      sources: "atmos_month_cmip"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      interpMethod: "conserve_order2"
+      timeSeries:
+        - freq: "3hr"
+          source: "atmos_8xdaily_cmip"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "daily"
+          source: "atmos_daily_cmip"
+          chunkLength: *PP_AMIP_CHUNK96
+          variables: "huss tasmin tasmax tas pr psl sfcWind hurs uas vas zg500 ua10 ua850 va850 ua200"
+        - freq: "monthly"
+          source: "atmos_month_refined"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source: "monthly"
+          interval: *ANA_AMIP_LEN
+        - source: "annual"
+          interval: *PP_AMIP_CHUNK96
+          calcInterval: *PP_AMIP_CHUNK96
+    - type: "atmos"
+      zInterp: "era40"
+      start: *ANA_AMIP_START
+      sources: "atmos_month"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      cmip: "off"
+      timeSeries:
+        - freq: "6hr"
+          source: "atmos_4xdaily"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source: "monthly"
+          interval: *ANA_AMIP_LEN
+        - source: "annual"
+          interval: *PP_AMIP_CHUNK96
+          calcInterval: *PP_AMIP_CHUNK96
+        - source: "annual"
+          interval: *ANA_AMIP_LEN
+    - type: "atmos_level_cmip"
+      start: *ANA_AMIP_START
+      sources: "atmos_level_cmip"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      cmip: "on"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+    - type: "atmos_level"
+      start: *ANA_AMIP_START
+      sources: "atmos_month"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      cmip: "off"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "annual"
+          chunkLength: *PP_AMIP_CHUNK96
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source: "monthly"
+          interval: *ANA_AMIP_LEN
+    - type: "atmos_month_aer"
+      start: *ANA_AMIP_START
+      sources: "atmos_month_aer"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      interpMethod: "conserve_order1"
+      cmip: "on"
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source:  "monthly"
+          interval: *ANA_AMIP_LEN
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+    - type: "atmos_diurnal"
+      start: *ANA_AMIP_START
+      sources: "atmos_diurnal"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+    - type: "atmos_scalar"
+      start: *ANA_AMIP_START
+      sources: "atmos_scalar"
+      cmip: "on"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "monthly"
+          source: "atmos_global_cmip"
+          chunkLength: *PP_AMIP_CHUNK96
+    - type: "tracer_level"
+      start: *ANA_AMIP_START
+      sources: "atmos_tracer"
+      sourceGrid: "atmos-cubedsphere"
+      xyInterp: *defaultxyInterp
+      interpMethod: "conserve_order1"
+      cmip: "on"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "annual"
+          chunkLength: *PP_AMIP_CHUNK96
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source: "monthly"
+          interval: *ANA_AMIP_LEN
+    - type: "aerosol_cmip"
+      start: *ANA_AMIP_START
+      xyInterp: *defaultxyInterp
+      sources: "aerosol_month_cmip"
+      sourceGrid: "atmos-cubedsphere"
+      interpMethod: "conserve_order1"
+      cmip: "on"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+          source: "aerosol_month_refined"
+    - type: "land"
+      start: *ANA_AMIP_START
+      sources: "land_month"
+      sourceGrid: "land-cubedsphere"
+      xyInterp: *defaultxyInterp
+      interpMethod: "conserve_order1"
+      cmip: "off"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+      timeAverage:
+        - source: "monthly"
+          interval: *PP_AMIP_CHUNK96
+        - source: "monthly"
+          interval: *ANA_AMIP_LEN
+    - type: "land_cmip"
+      start: *ANA_AMIP_START
+      sources: "land_month_cmip"
+      sourceGrid: "land-cubedsphere"
+      xyInterp: *defaultxyInterp
+      interpMethod: "conserve_order1"
+      cmip: "off"
+      timeSeries:
+        - freq: "monthly"
+          chunkLength: *PP_AMIP_CHUNK96
+
+tmpdir:
+  tmpdirpath:
+
+install-exp-script:
+    - path: 
+    - install-option:
+        install: