Merge branch 'main' into github_action_exp

PCMDI · May 3, 2024 · 4e2b014 · 4e2b014
2 parents 5eefb7d + 92e3efd
commit 4e2b014
Show file tree

Hide file tree

Showing 22 changed files with 1,252 additions and 46 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "cmip6-cmor-tables"]
 	path = cmip6-cmor-tables
 	url = https://github.com/PCMDI/cmip6-cmor-tables.git
+[submodule "mip-cmor-tables"]
+	path = mip-cmor-tables
+	url = https://github.com/PCMDI/mip-cmor-tables.git
diff --git a/.zenodo.json b/.zenodo.json
@@ -2,10 +2,10 @@
     "description": "<p>Climate Model Output Rewriter</p>",
     "access_right": "open",
     "license": "BSD-3-Clause",
-    "title": "PCMDI/CMOR: CMOR v3.7.3",
-    "version": "v3.7.3",
+    "title": "PCMDI/CMOR: CMOR v3.8.0",
+    "version": "v3.8.0",
     "upload_type": "software",
-    "publication_date": "2023-10-17",
+    "publication_date": "2024-04-08",
     "creators": [
         {
             "name": "Mauzey, Chris",

diff --git a/INSTALL b/INSTALL
@@ -1,6 +1,6 @@
 INSTALLATION INSTRUCTIONS  
 ------------------------- 
-Climate Model Output Rewriter (CMOR) version 3.7.3 installation instructions.
+Climate Model Output Rewriter (CMOR) version 3.8.0 installation instructions.
 
 
 DOWNLOAD 
@@ -13,7 +13,7 @@ cd CMOR
 
 INSTALLATION
 ------------ 
-CMOR 3.7.3 requires external packages that need to be installed first. 
+CMOR 3.8.0 requires external packages that need to be installed first. 
 It can be compiled/linked against either NetCDF3 or NetCDF4. If you
 decide to go with NetCDF4 be sure to build NetCDF4 with the
  --enable-netcdf-4 option! 

diff --git a/Makefile.in b/Makefile.in
@@ -224,6 +224,7 @@ test_python: python
 	env TEST_NAME=Test/test_cmor_python_not_enough_data.py make test_a_python
 	env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python
 	env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python
+	env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python
 test_cmip6_cv: python
 	env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python
 	env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python

diff --git a/README.md b/README.md
@@ -1,21 +1,21 @@
 # CMOR
 Climate Model Output Rewriter
 
-CMOR 3.7 documentation can be found at: http://cmor.llnl.gov
+CMOR3 documentation can be found at: http://cmor.llnl.gov
 
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10044293.svg)](https://doi.org/10.5281/zenodo.10044293)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10946710.svg)](https://doi.org/10.5281/zenodo.10946710)
 [![stable version](https://img.shields.io/github/v/release/pcmdi/cmor.svg)](https://github.com/PCMDI/cmor/releases/latest)
 ![platforms](https://anaconda.org/pcmdi/cmor/badges/platforms.svg)
-[![Anaconda-Server Badge](https://anaconda.org/pcmdi/cmor/badges/version.svg)](https://conda.anaconda.org/pcmdi)
-[![Anaconda-Server Badge](https://anaconda.org/pcmdi/cmor/badges/downloads.svg)](https://anaconda.org/pcmdi)
+[![Anaconda-Server Badge](https://anaconda.org/conda-forge/cmor/badges/version.svg)](https://anaconda.org/conda-forge/cmor)
+[![Anaconda-Server Badge](https://anaconda.org/conda-forge/cmor/badges/downloads.svg)](https://anaconda.org/conda-forge/cmor)
 
 ## Contributors
 
 [![Contributors](https://contrib.rocks/image?repo=PCMDI/cmor)](https://github.com/PCMDI/cmor/graphs/contributors)
 
 ## Acknowledgement
 
-The repository content is developed by climate and computer scientists from the Program for Climate Model Diagnosis and Intercomparison ([PCMDI][PCMDI]) at Lawrence Livermore National Laboratory ([LLNL][LLNL]), with contributions from colleagues around the world. This work is sponsored by the Regional and Global Model Analysis ([RGMA][RGMA]) program, of the Earth and Environmental Systems Sciences Division ([EESSD][EESSD]) in the Office of Biological and Environmental Research ([BER][BER]) within the [Department of Energy][DOE]'s [Office of Science][OS]. The work is performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344.
+The content of this repository is developed by climate and computer scientists from the Program for Climate Model Diagnosis and Intercomparison ([PCMDI][PCMDI]) at Lawrence Livermore National Laboratory ([LLNL][LLNL]), with contributions from colleagues around the world. This work is sponsored by the Regional and Global Model Analysis ([RGMA][RGMA]) program, of the Earth and Environmental Systems Sciences Division ([EESSD][EESSD]) in the Office of Biological and Environmental Research ([BER][BER]) within the [Department of Energy][DOE]'s [Office of Science][OS]. The work is performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344.
 
 <p>
     <img src="https://pcmdi.github.io/assets/PCMDI/100px-PCMDI-Logo-NoText-square-png8.png"

diff --git a/Src/cmor.c b/Src/cmor.c
@@ -1321,7 +1321,7 @@ int cmor_dataset_json(char *ressource)
                 CMOR_DEFAULT_FILE_TEMPLATE, CMOR_MAX_STRING);
 
     strncpytrim(cmor_current_dataset.furtherinfourl,
-                CMOR_DEFAULT_FURTHERURL_TEMPLATE, CMOR_MAX_STRING);
+                "", CMOR_MAX_STRING);
 
     strncpytrim(cmor_current_dataset.history_template,
                 CMOR_DEFAULT_HISTORY_TEMPLATE, CMOR_MAX_STRING);
@@ -1375,6 +1375,7 @@ int cmor_dataset_json(char *ressource)
         } else if (strcmp(key, GLOBAL_ATT_FURTHERINFOURL) == 0) {
             strncpytrim(cmor_current_dataset.furtherinfourl,
                         szVal, CMOR_MAX_STRING);
+            continue;
         }
         cmor_set_cur_dataset_attribute_internal(key, szVal, 1);
     }
@@ -2794,14 +2795,27 @@ int cmor_setDefaultGblAttr(int ref_table_id)
         if(cmor_has_cur_dataset_attribute(CV_value->key) != 0){
             if(CV_value->szValue[0] != '\0'){
                 ierr |= cmor_set_cur_dataset_attribute_internal(CV_value->key, CV_value->szValue, 0);
-                if(strncmp(CV_value->key, GLOBAL_ATT_FURTHERINFOURL, CMOR_MAX_STRING) == 0){
-                    strncpytrim(cmor_current_dataset.furtherinfourl, CV_value->szValue, CMOR_MAX_STRING);
+                if(strncmp(CV_value->key, GLOBAL_ATT_FURTHERINFOURL, CMOR_MAX_STRING) == 0
+                     && cmor_current_dataset.furtherinfourl[0] == '\0'){
+                    ierr |= cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_FURTHERINFOURLTMPL, CV_value->szValue, 0);
                 }
             } else if(CV_value->anElements == 1 && isRequired == 1){
                 ierr |= cmor_set_cur_dataset_attribute_internal(CV_value->key, CV_value->aszValue[0], 0);
             }
         }
     }
+
+/* -------------------------------------------------------------------- */
+/*  Set further_info_url template if required and not already set.      */
+/* -------------------------------------------------------------------- */
+    for (k = 0; k < required_attrs->anElements; k++) {
+        if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_FURTHERINFOURL) == 0
+            && cmor_current_dataset.furtherinfourl[0] == '\0')
+        {
+            ierr |= cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_FURTHERINFOURLTMPL, CMOR_DEFAULT_FURTHERURL_TEMPLATE, 0);
+        }
+    }
+
     cmor_pop_traceback();
     return ierr;
 
@@ -2939,16 +2953,10 @@ int cmor_setGblAttr(int var_id)
 /*      first check if the variable itself has a realm                  */
 /* -------------------------------------------------------------------- */
     if (cmor_tables[nVarRefTblID].vars[ref_var_id].realm[0] != '\0') {
-        szToken = strtok(cmor_tables[nVarRefTblID].vars[ref_var_id].realm, " ");
-        if (szToken != NULL) {
-            cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_REALM,
-                                                    szToken, 0);
-        } else {
-            cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_REALM,
-                                                    cmor_tables
-                                                    [nVarRefTblID].vars
-                                                    [ref_var_id].realm, 0);
-        }
+        cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_REALM,
+                                                cmor_tables
+                                                [nVarRefTblID].vars
+                                                [ref_var_id].realm, 0);
     } else {
 /* -------------------------------------------------------------------- */
 /*      ok it didn't so we're using the value from the table            */
@@ -3039,11 +3047,11 @@ int cmor_setGblAttr(int var_id)
         ierr += cmor_CV_setInstitution(cmor_tables[nVarRefTblID].CV);
     }
 
+    ierr += cmor_CV_checkFurtherInfoURL(nVarRefTblID);
 
     if (cmor_has_cur_dataset_attribute(GLOBAL_IS_CMIP6) == 0) {
         ierr += cmor_CV_checkSourceID(cmor_tables[nVarRefTblID].CV);
         ierr += cmor_CV_checkExperiment(cmor_tables[nVarRefTblID].CV);
-        ierr += cmor_CV_checkFurtherInfoURL(nVarRefTblID);
         ierr += cmor_CV_checkGrids(cmor_tables[nVarRefTblID].CV);
         ierr += cmor_CV_checkParentExpID(cmor_tables[nVarRefTblID].CV);
         ierr += cmor_CV_checkSubExpID(cmor_tables[nVarRefTblID].CV);
@@ -3058,7 +3066,6 @@ int cmor_setGblAttr(int var_id)
     //
     if ( cmor_current_dataset.furtherinfourl[0] != '\0') {
         ierr += cmor_CV_checkSourceID(cmor_tables[nVarRefTblID].CV);
-        ierr += cmor_CV_checkFurtherInfoURL(nVarRefTblID);
     }
 
     ierr += cmor_CV_checkISOTime(GLOBAL_ATT_CREATION_DATE);
@@ -3581,7 +3588,7 @@ void cmor_define_dimensions(int var_id, int ncid,
                 && (CMOR_NETCDF_MODE != CMOR_APPEND_3)) {
                 if (strcmp(pAxis->id, "time") == 0) {
                     ierr = nc_def_var_chunking(ncid, nc_vars[i], NC_CHUNKED,
-                                               &nc_dim_chunking[0]);
+                                               NULL);
                 } else {
                     ierr = nc_def_var_chunking(ncid, nc_vars[i], NC_CONTIGUOUS,
                                                &nc_dim_chunking[0]);
@@ -5189,7 +5196,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
                                 1)))) {
             ierr =
               nc_def_var_chunking(ncid, cmor_vars[var_id].nc_var_id, NC_CHUNKED,
-                                  &nc_dim_chunking[0]);
+                                 NULL);
             if (ierr != NC_NOERR) {
                 snprintf(msg, CMOR_MAX_STRING,
                          "NetCDFTestTables/CMIP6_chunking.json: Error (%i: %s) defining chunking\n! "

diff --git a/Src/cmor_CV.c b/Src/cmor_CV.c
@@ -370,6 +370,13 @@ int cmor_CV_checkFurtherInfoURL(int nVarRefTblID)
     cmor_is_setup();
     cmor_add_traceback("_CV_checkFurtherInfoURL");
 
+/* -------------------------------------------------------------------- */
+/* If the template is an emtpy string, then skip this check.            */
+/* -------------------------------------------------------------------- */
+    if (cmor_current_dataset.furtherinfourl[0] == '\0') {
+        return (0);
+    }
+
 /* -------------------------------------------------------------------- */
 /* Retrieve default Further URL info                                    */
 /* -------------------------------------------------------------------- */
@@ -391,8 +398,11 @@ int cmor_CV_checkFurtherInfoURL(int nVarRefTblID)
     }
 
     if (strcmp(szToken, cmor_current_dataset.furtherinfourl) == 0) {
+        cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_FURTHERINFOURL,
+                                                cmor_current_dataset.furtherinfourl, 0);
         return (0);
     }
+
     strncpy(szFurtherInfoURLTemplate, cmor_current_dataset.furtherinfourl,
             CMOR_MAX_STRING);
 /* -------------------------------------------------------------------- */

diff --git a/Src/cmor_fortran_interface.f90 b/Src/cmor_fortran_interface.f90
@@ -806,7 +806,7 @@ end function cmor_close_cff
   integer, parameter:: CMOR_EXIT_ON_MAJOR = 0
   integer, parameter:: CMOR_EXIT_ON_WARNING = 2
 
-  real,    parameter:: CMOR_VERSION = 3.7
+  real,    parameter:: CMOR_VERSION = 3.8
   real,    parameter:: CMOR_CF_VERSION = 1.7
 
   integer, parameter:: CMOR_WARNING = 20

diff --git a/Src/cmor_tables.c b/Src/cmor_tables.c
@@ -9,6 +9,7 @@
 #include "cmor_locale.h"
 #include <json-c/json.h>
 #include <json-c/json_tokener.h>
+#include <json-c/arraylist.h>
 #include <sys/stat.h>
 
 /************************************************************************/
@@ -171,6 +172,9 @@ int cmor_set_variable_entry(cmor_table_t * table,
     char msg[CMOR_MAX_STRING];
     int nVarId;
     char *szTableId;
+    array_list *jsonArray;
+    json_object *jsonItem;
+    size_t k, arrayLen;
     cmor_var_def_t *variable;
     cmor_table_t *cmor_table;
     cmor_table = &cmor_tables[cmor_ntables];
@@ -208,7 +212,26 @@ int cmor_set_variable_entry(cmor_table_t * table,
         if (attr[0] == '#')
             continue;
 
-        strcpy(szValue, json_object_get_string(value));
+/* -------------------------------------------------------------------- */
+/*  Attribute values that are arrays will have their array elements     */
+/*  combined into space-separated lists.                                */
+/* -------------------------------------------------------------------- */
+        if(json_object_is_type(value, json_type_array)) {
+            jsonArray = json_object_get_array(value);
+            arrayLen = array_list_length(jsonArray);
+            for (k = 0; k < arrayLen; k++) {
+                jsonItem = (json_object *) array_list_get_idx(jsonArray, k);
+                if (k == 0) {
+                    strcpy(szValue, json_object_get_string(jsonItem));
+                } else {
+                    strcat(szValue, " ");
+                    strcat(szValue, json_object_get_string(jsonItem));
+                }
+            }
+        } else {
+            strcpy(szValue, json_object_get_string(value));
+        }
+
         cmor_set_var_def_att(variable, attr, szValue);
     }
     cmor_pop_traceback();
@@ -772,6 +795,18 @@ int cmor_load_table_internal(char szTable[CMOR_MAX_STRING], int *table_id)
         return (TABLE_ERROR);
     }
 
+/* -------------------------------------------------------------------- */
+/*     check for null values in JSON                                    */
+/* -------------------------------------------------------------------- */
+    if(cmor_validate_json(json_obj) != 0) {
+        snprintf(msg, CMOR_MAX_STRING,
+                 "There are invalid null values in table: %s", 
+                 szTable);
+        cmor_handle_error(msg, CMOR_CRITICAL);
+        cmor_pop_traceback();
+        return (TABLE_ERROR);
+    }
+
     json_object_object_foreach(json_obj, key, value) {
 
         if (key[0] == '#') {
@@ -985,3 +1020,37 @@ int cmor_load_table_internal(char szTable[CMOR_MAX_STRING], int *table_id)
     json_object_put(json_obj);
     return (TABLE_SUCCESS);
 }
+
+
+/************************************************************************/
+/*                      cmor_validate_json()                            */
+/************************************************************************/
+int cmor_validate_json(json_object *json)
+{
+    json_object *value;
+    array_list *array;
+    size_t length, k;
+
+    if (json_object_is_type(json, json_type_null)) {
+        // null is invalid
+        return 1;
+    } else if (json_object_is_type(json, json_type_object)) {
+        // validate values within JSON object
+        json_object_object_foreach(json, key, value) {
+            if (cmor_validate_json(value) == 1)
+                return 1;
+        }
+    } else if (json_object_is_type(json, json_type_array)) {
+        // validate values within JSON list
+        array = json_object_get_array(json);
+        length = array_list_length(array);
+        for (k = 0; k < length; k++) {
+            value = (json_object *) array_list_get_idx(array, k);
+            if (cmor_validate_json(value) == 1)
+                return 1;
+        }
+    }
+
+    return 0;
+
+}
diff --git a/Test/CMIP6Plus_user_input.json b/Test/CMIP6Plus_user_input.json
@@ -0,0 +1,39 @@
+{
+    "branch_method": "standard",
+    "calendar": "360_day",
+    "experiment_id": "hist-lu",
+    "institution_id": "MOHC",
+    "license": "CMIP6Plus model data produced by the Met Office Hadley Centre is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6Plus/TermsOfUse for terms of use governing CMIP6Plus output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.",
+    "activity_id": "LESFMIP",
+    "mip_era": "CMIP6Plus",
+    "source_id": "HadGEM3-GC31-LL",
+    "source_type": "AOGCM BGC AER CHEM",
+    "outpath": ".",
+    "sub_experiment_id": "f2023",
+    "variant_label": "r1i1p1f2",
+    "grid": "Native N96 grid; 192 x 144 longitude/latitude",
+    "grid_label": "gn",
+    "nominal_resolution": "250 km",
+    "parent_experiment_id": "piControl",
+    "parent_mip_era": "CMIP6Plus",
+    "parent_source_id": "HadGEM3-GC31-LL",
+    "parent_time_units": "days since 1850-01-01",
+    "parent_variant_label": "r1i1p1f2",
+    "history": "2023-11-03T11:41:57Z MIP Convert v2.6.0.dev0, Python v3.8.17, iris v3.4.1, numpy v1.24.4, cftime v1.6.2, pyproj v3.5.0",
+    "#_cmip6_option": "CMIP6",
+    "realization_index": "1",
+    "initialization_index": "1",
+    "physics_index": "1",
+    "forcing_index": "2",
+    "cv_version": "0.0.1",
+    "experiment": "Single forcing historical simulations",
+    "sub_experiment": "Forcings 2023",
+    "tracking_prefix": "hdl:21.14100",
+    "parent_activity_id": "CMIP",
+    "_controlled_vocabulary_file": "TestTables/CMIP6Plus_CV.json",
+    "_AXIS_ENTRY_FILE": "mip-cmor-tables/Auxillary_files/MIP_coordinate.json",
+    "_FORMULA_VAR_FILE": "mip-cmor-tables/Auxillary_files/MIP_formula_terms.json",
+    "mo_runid": "u-be509",
+    "branch_time_in_parent": 59400.0,
+    "branch_time_in_child": 59400.0
+}
diff --git a/Test/CMOR_input_example_badfurtherinfourl.json b/Test/CMOR_input_example_badfurtherinfourl.json
@@ -64,7 +64,7 @@
 
     "mip_era":                "CMIP6",
     "parent_mip_era":         "CMIP6",
-    "further_info_url":       "",
+    "further_info_url":       "bad_url",
 
     "tracking_prefix":        "hdl:21.14100",
     "_history_template":       "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",