From 2efc3e00dfd8b4b674273706b35942f038111717 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Wed, 25 Oct 2023 16:21:56 -0600 Subject: [PATCH 01/10] setext - porting setext and extfrc_set --- src/mam4xx/mam4.hpp | 1 + src/mam4xx/mo_setext.hpp | 134 ++++++++++++++++++++++++++++++++ src/validation/mam_x_validation | 2 +- 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 src/mam4xx/mo_setext.hpp diff --git a/src/mam4xx/mam4.hpp b/src/mam4xx/mam4.hpp index bd953d7df..ccd67dd2a 100644 --- a/src/mam4xx/mam4.hpp +++ b/src/mam4xx/mam4.hpp @@ -30,6 +30,7 @@ #include #include #include +#include namespace mam4 { diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp new file mode 100644 index 000000000..0cbcdd865 --- /dev/null +++ b/src/mam4xx/mo_setext.hpp @@ -0,0 +1,134 @@ +#ifndef MAM4XX_MO_SETEXT_HPP +#define MAM4XX_MO_SETEXT_HPP + +#include +#include +#include +#include + +namespace mam4 { + +namespace mo_setext { +using View2D = DeviceType::view_2d; +constexpr int pver = mam4::nlev; +// FIXME get this constant +constexpr int extfrc_cnt = 1; +// FIXME check if this constant is defined somewhere else. +constexpr int extcnt = 9; //, & ! number of species with external forcing + +KOKKOS_INLINE_FUNCTION +void extfrc_set(const ColumnView &zint, const int *forcings_frc_ndx, + const int *forcings_nsectors, + const bool *forcings_file_alt_data, + const View2D &forcings_fields_data, const View2D &frcing) { + /*-------------------------------------------------------- + ! ... form the external forcing + !--------------------------------------------------------*/ + + // ncol ! columns in chunk + // lchnk ! chunk index + // zint(ncol, pverp) ! interface geopot above surface [km] + // frcing(ncol,pver,extcnt) ! insitu forcings [molec/cm^3/s] + constexpr Real zero = 0.0; + + // frcing(:,:,:) = zero; + if (extfrc_cnt < 1 || extcnt < 1) { + return; + } + + /*-------------------------------------------------------- + ! ... set non-zero forcings + !--------------------------------------------------------*/ + + for (int mm = 0; mm < extfrc_cnt; ++mm) { + const int nn = forcings_frc_ndx[mm]; + for (int kk = 0; kk < pver; ++kk) { + frcing(kk, nn) = zero; + } // k + + for (int isec = 0; isec < forcings_nsectors[mm]; ++isec) { + if (forcings_file_alt_data[mm]) { + // FIXME: forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) + // Note pver:1:-1 + for (int kk = pver; kk < -1; --kk) { + frcing(kk, nn) += forcings_fields_data(isec, kk); + } // kk + } else { + // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) + for (int kk = 0; kk < pver; ++kk) { + frcing(kk, nn) += forcings_fields_data(isec, kk); + } + } + } // isec + + // xfcname = trim(forcings(mm)%species)//'_XFRC' + // call outfld( xfcname, frcing(:ncol,:,nn), ncol, lchnk ) + + // frcing_col(:ncol) = 0._r8 + // do kk = 1,pver + // frcing_col(:ncol) = frcing_col(:ncol) + & + // frcing(:ncol,kk,nn)*(zint(:ncol,kk)-zint(:ncol,kk+1))*km_to_cm + // enddo + // xfcname = trim(forcings(mm)%species)//'_CLXF' + // call outfld( xfcname, frcing_col(:ncol), ncol, lchnk ) + + } // end mm +} // extfrc_set + +KOKKOS_INLINE_FUNCTION +void setext(const ColumnView &zint, const int *forcings_frc_ndx, + const int *forcings_nsectors, const bool *forcings_file_alt_data, + const View2D &forcings_fields_data, + const View2D &extfrc) // ! out +{ + + /*-------------------------------------------------------- + ! ... for this latitude slice: + ! - form the production from datasets + ! - form the nox (xnox) production from lighing + ! - form the nox (xnox) production from airplanes + ! - form the co production from airplanes + !--------------------------------------------------------*/ + + // @param[in] zint(ncol,pver+1) ! interface geopot height [km] + // @param[out] extfrc(ncol,pver,extcnt) ! the "extraneous" forcing + + /*-------------------------------------------------------- + ! ... local variables + !-------------------------------------------------------- + ! variables for output. in current MAM4 they are not calculated and are + assigned zero real(r8), dimension(ncol,pver) :: no_lgt, no_air, co_air */ + + /*-------------------------------------------------------- + ! ... set frcing from datasets + !--------------------------------------------------------*/ + extfrc_set(zint, // in + forcings_frc_ndx, forcings_nsectors, forcings_file_alt_data, + forcings_fields_data, + extfrc); // out + + /*-------------------------------------------------------- + ! ... set nox production from lighting + ! note: from ground to cloud top production is c shaped + ! + ! FORTRAN refactor: nox is not included in current MAM4 + ! the related code are removed but outfld is kept for BFB testing + !--------------------------------------------------------*/ + // no_lgt(:,:) = 0._r8 + // call outfld( 'NO_Lightning', no_lgt(:ncol,:), ncol, lchnk ) + + // ! FORTRAN refactor: in the subroutine airpl_set, has_airpl_src is false, + // ! the subroutine only has two outfld calls that output zero + // ! remove the subroutine call and move out the zero outfld + // no_air(:,:) = 0._r8 + // co_air(:,:) = 0._r8 + // call outfld( 'NO_Aircraft', no_air(:ncol,:), ncol, lchnk ) + // call outfld( 'CO_Aircraft', co_air(:ncol,:), ncol, lchnk ) + +} // setext + +} // namespace mo_setext + +} // end namespace mam4 + +#endif \ No newline at end of file diff --git a/src/validation/mam_x_validation b/src/validation/mam_x_validation index 5867a0c0d..a35ada82e 160000 --- a/src/validation/mam_x_validation +++ b/src/validation/mam_x_validation @@ -1 +1 @@ -Subproject commit 5867a0c0dffc9c1591e278046c46523e03cd2edf +Subproject commit a35ada82ea68df5b3d92f53d343f310ec36d1914 From cc2e3299f5276f2c696c5c7d27c7ebb091f62705 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 16:36:14 -0600 Subject: [PATCH 02/10] mo_setext - validation test for extfrc_set --- src/mam4xx/mo_setext.hpp | 31 +++++++++++++++++++------------ src/validation/CMakeLists.txt | 1 + 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index 0cbcdd865..c43156f33 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -9,18 +9,21 @@ namespace mam4 { namespace mo_setext { + using View1D = DeviceType::view_1d; using View2D = DeviceType::view_2d; +using View3D = DeviceType::view_3d; constexpr int pver = mam4::nlev; // FIXME get this constant -constexpr int extfrc_cnt = 1; +constexpr int extfrc_cnt = 9; // FIXME check if this constant is defined somewhere else. constexpr int extcnt = 9; //, & ! number of species with external forcing KOKKOS_INLINE_FUNCTION -void extfrc_set(const ColumnView &zint, const int *forcings_frc_ndx, +void extfrc_set( + const int *forcings_frc_ndx, const int *forcings_nsectors, const bool *forcings_file_alt_data, - const View2D &forcings_fields_data, const View2D &frcing) { + const View1D forcings_fields_data[extfrc_cnt][4], const View2D &frcing) { /*-------------------------------------------------------- ! ... form the external forcing !--------------------------------------------------------*/ @@ -29,6 +32,9 @@ void extfrc_set(const ColumnView &zint, const int *forcings_frc_ndx, // lchnk ! chunk index // zint(ncol, pverp) ! interface geopot above surface [km] // frcing(ncol,pver,extcnt) ! insitu forcings [molec/cm^3/s] + // Note: we do not need zint to compute frcing + // const ColumnView &zint, + constexpr Real zero = 0.0; // frcing(:,:,:) = zero; @@ -41,22 +47,23 @@ void extfrc_set(const ColumnView &zint, const int *forcings_frc_ndx, !--------------------------------------------------------*/ for (int mm = 0; mm < extfrc_cnt; ++mm) { - const int nn = forcings_frc_ndx[mm]; + // Fortran to C++ indexing + const int nn = forcings_frc_ndx[mm]-1; for (int kk = 0; kk < pver; ++kk) { frcing(kk, nn) = zero; } // k for (int isec = 0; isec < forcings_nsectors[mm]; ++isec) { if (forcings_file_alt_data[mm]) { - // FIXME: forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) - // Note pver:1:-1 - for (int kk = pver; kk < -1; --kk) { - frcing(kk, nn) += forcings_fields_data(isec, kk); + for (int kk = 0; kk < pver; ++kk) { + // frcing(:ncol,:,nn) = frcing(:ncol,:,nn) + & + // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) + frcing(kk, nn) += forcings_fields_data[mm][isec](pver-1-kk); } // kk } else { // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) for (int kk = 0; kk < pver; ++kk) { - frcing(kk, nn) += forcings_fields_data(isec, kk); + frcing(kk, nn) += forcings_fields_data[mm][isec](kk); } } } // isec @@ -76,9 +83,9 @@ void extfrc_set(const ColumnView &zint, const int *forcings_frc_ndx, } // extfrc_set KOKKOS_INLINE_FUNCTION -void setext(const ColumnView &zint, const int *forcings_frc_ndx, +void setext(const int *forcings_frc_ndx, const int *forcings_nsectors, const bool *forcings_file_alt_data, - const View2D &forcings_fields_data, + const View1D forcings_fields_data[extfrc_cnt][4], const View2D &extfrc) // ! out { @@ -102,7 +109,7 @@ void setext(const ColumnView &zint, const int *forcings_frc_ndx, /*-------------------------------------------------------- ! ... set frcing from datasets !--------------------------------------------------------*/ - extfrc_set(zint, // in + extfrc_set( forcings_frc_ndx, forcings_nsectors, forcings_file_alt_data, forcings_fields_data, extfrc); // out diff --git a/src/validation/CMakeLists.txt b/src/validation/CMakeLists.txt index 0f80accf9..eb2afe940 100644 --- a/src/validation/CMakeLists.txt +++ b/src/validation/CMakeLists.txt @@ -31,3 +31,4 @@ add_subdirectory(water_uptake) add_subdirectory(mo_photo) add_subdirectory(lin_strat_chem) add_subdirectory(mo_chm_diags) +add_subdirectory(mo_setext) From 154ba47ac23a51c9905c4106fd21916a2e521d03 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 16:54:22 -0600 Subject: [PATCH 03/10] mo_setext - clang format --- src/mam4xx/mam4.hpp | 2 +- src/mam4xx/mo_setext.hpp | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/mam4xx/mam4.hpp b/src/mam4xx/mam4.hpp index ccd67dd2a..e5833e0fc 100644 --- a/src/mam4xx/mam4.hpp +++ b/src/mam4xx/mam4.hpp @@ -24,13 +24,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include namespace mam4 { diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index c43156f33..471cf0ac3 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -9,7 +9,7 @@ namespace mam4 { namespace mo_setext { - using View1D = DeviceType::view_1d; +using View1D = DeviceType::view_1d; using View2D = DeviceType::view_2d; using View3D = DeviceType::view_3d; constexpr int pver = mam4::nlev; @@ -19,11 +19,10 @@ constexpr int extfrc_cnt = 9; constexpr int extcnt = 9; //, & ! number of species with external forcing KOKKOS_INLINE_FUNCTION -void extfrc_set( - const int *forcings_frc_ndx, - const int *forcings_nsectors, +void extfrc_set(const int *forcings_frc_ndx, const int *forcings_nsectors, const bool *forcings_file_alt_data, - const View1D forcings_fields_data[extfrc_cnt][4], const View2D &frcing) { + const View1D forcings_fields_data[extfrc_cnt][4], + const View2D &frcing) { /*-------------------------------------------------------- ! ... form the external forcing !--------------------------------------------------------*/ @@ -33,7 +32,7 @@ void extfrc_set( // zint(ncol, pverp) ! interface geopot above surface [km] // frcing(ncol,pver,extcnt) ! insitu forcings [molec/cm^3/s] // Note: we do not need zint to compute frcing - // const ColumnView &zint, + // const ColumnView &zint, constexpr Real zero = 0.0; @@ -48,17 +47,17 @@ void extfrc_set( for (int mm = 0; mm < extfrc_cnt; ++mm) { // Fortran to C++ indexing - const int nn = forcings_frc_ndx[mm]-1; + const int nn = forcings_frc_ndx[mm] - 1; for (int kk = 0; kk < pver; ++kk) { frcing(kk, nn) = zero; } // k for (int isec = 0; isec < forcings_nsectors[mm]; ++isec) { if (forcings_file_alt_data[mm]) { - for (int kk = 0; kk < pver; ++kk) { + for (int kk = 0; kk < pver; ++kk) { // frcing(:ncol,:,nn) = frcing(:ncol,:,nn) + & - // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) - frcing(kk, nn) += forcings_fields_data[mm][isec](pver-1-kk); + // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) + frcing(kk, nn) += forcings_fields_data[mm][isec](pver - 1 - kk); } // kk } else { // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) @@ -83,8 +82,8 @@ void extfrc_set( } // extfrc_set KOKKOS_INLINE_FUNCTION -void setext(const int *forcings_frc_ndx, - const int *forcings_nsectors, const bool *forcings_file_alt_data, +void setext(const int *forcings_frc_ndx, const int *forcings_nsectors, + const bool *forcings_file_alt_data, const View1D forcings_fields_data[extfrc_cnt][4], const View2D &extfrc) // ! out { @@ -109,8 +108,7 @@ void setext(const int *forcings_frc_ndx, /*-------------------------------------------------------- ! ... set frcing from datasets !--------------------------------------------------------*/ - extfrc_set( - forcings_frc_ndx, forcings_nsectors, forcings_file_alt_data, + extfrc_set(forcings_frc_ndx, forcings_nsectors, forcings_file_alt_data, forcings_fields_data, extfrc); // out From cd55c56c31ac0389b82ac69b04934c47e4f64bab Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 16:56:37 -0600 Subject: [PATCH 04/10] mo_setext - update mam_x_validation --- src/validation/mam_x_validation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validation/mam_x_validation b/src/validation/mam_x_validation index a35ada82e..eb1901c1f 160000 --- a/src/validation/mam_x_validation +++ b/src/validation/mam_x_validation @@ -1 +1 @@ -Subproject commit a35ada82ea68df5b3d92f53d343f310ec36d1914 +Subproject commit eb1901c1fe13feff81a91a9a8e5b0e02e48aa643 From 3357a54f0570759d190570b946f54bd1d61ca00c Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 17:01:36 -0600 Subject: [PATCH 05/10] mo_setext - missing directory --- src/validation/mo_setext/CMakeLists.txt | 55 +++++++++++++++++ src/validation/mo_setext/extfrc_set.cpp | 65 +++++++++++++++++++++ src/validation/mo_setext/setext_driver.cpp | 68 ++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 src/validation/mo_setext/CMakeLists.txt create mode 100644 src/validation/mo_setext/extfrc_set.cpp create mode 100644 src/validation/mo_setext/setext_driver.cpp diff --git a/src/validation/mo_setext/CMakeLists.txt b/src/validation/mo_setext/CMakeLists.txt new file mode 100644 index 000000000..f00fd0b35 --- /dev/null +++ b/src/validation/mo_setext/CMakeLists.txt @@ -0,0 +1,55 @@ +set(SETEXT_VALIDATION_DIR ${MAM_X_VALIDATION_DIR}/mo_setext) +set(SETEXT_VALIDATION_SCRIPTS_DIR ${MAM_X_VALIDATION_DIR}/scripts) + + +# These subdirectories contain Skywalker drivers for MAM4 parameterizations. +# Include directory for .mod files. + +include_directories(${PROJECT_BINARY_DIR}/validation) + +# We use a single driver for all setext-related parameterizations. +add_executable(setext_driver + setext_driver.cpp + extfrc_set.cpp + ) + +target_link_libraries(setext_driver skywalker;validation;${HAERO_LIBRARIES}) + +# Copy some Python scripts from mam_x_validation to our binary directory. +foreach(script + compare_mam4xx_mam4.py) + configure_file( + ${SETEXT_VALIDATION_SCRIPTS_DIR}/${script} + ${CMAKE_CURRENT_BINARY_DIR}/${script} + COPYONLY + ) +endforeach() +# stand_calc_sum_wght_ts_355 +# Run the driver in several configurations to produce datasets. +set(TEST_LIST + extfrc_set_ts_355 + ) +# # matching the tests and errors, just for convenience + +set(DEFAULT_TOL 1e-13) +set(ERROR_THRESHOLDS +9e-10 + ) +foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS) + # copy the baseline file into place. + + configure_file( + ${SETEXT_VALIDATION_DIR}/mam_${input}.py + ${CMAKE_CURRENT_BINARY_DIR}/mam_${input}.py + COPYONLY + ) + + # add a test to run the skywalker driver + add_test(run_${input} setext_driver ${SETEXT_VALIDATION_DIR}/${input}.yaml) + + # add a test to validate mam4xx's results against the baseline. + # Select a threshold error slightly bigger than the largest relative error for the threshold error. + # compare_mam4xx_mam4.py + add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol}) + set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input}) +endforeach() diff --git a/src/validation/mo_setext/extfrc_set.cpp b/src/validation/mo_setext/extfrc_set.cpp new file mode 100644 index 000000000..28956d1ab --- /dev/null +++ b/src/validation/mo_setext/extfrc_set.cpp @@ -0,0 +1,65 @@ +// mam4xx: Copyright (c) 2022, +// Battelle Memorial Institute and +// National Technology & Engineering Solutions of Sandia, LLC (NTESS) +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include +#include + +using namespace skywalker; +using namespace mam4; +using namespace haero; +using namespace mo_setext; +void extfrc_set(Ensemble *ensemble) { + ensemble->process([=](const Input &input, Output &output) { + using View1DHost = typename HostType::view_1d; + + int forcings_frc_ndx[extfrc_cnt] = {}; + int forcings_nsectors[extfrc_cnt] = {}; + // std::vector> data; + + bool forcings_file_alt_data[extfrc_cnt] = {}; + View1DHost forcings_fields_data_host; + View1D forcings_fields_data[extfrc_cnt][4]; + for (int i = 1; i <= extfrc_cnt; ++i) { + forcings_frc_ndx[i - 1] = + int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); + forcings_nsectors[i - 1] = + int(input.get_array("forcings" + std::to_string(i) + "_nsectors")[0]); + const int file_alt_data = int(input.get_array( + "forcings" + std::to_string(i) + "_file_alt_data")[0]); + forcings_file_alt_data[i - 1] = file_alt_data; + for (int isec = 1; isec <= forcings_nsectors[i - 1]; ++isec) { + + auto label = "forcings" + std::to_string(i) + "_fields" + + std::to_string(isec) + "_data"; + const auto data1 = input.get_array(label); + + forcings_fields_data_host = + View1DHost((Real *)data1.data(), data1.size()); + forcings_fields_data[i - 1][isec - 1] = + View1D("forcings_fields_data", data1.size()); + + Kokkos::deep_copy(forcings_fields_data[i - 1][isec - 1], + forcings_fields_data_host); + } + } + + View2D frcing("frcing", pver, extcnt); + const int ncol = 1; + auto team_policy = ThreadTeamPolicy(ncol, 1u); + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + extfrc_set(forcings_frc_ndx, forcings_nsectors, + forcings_file_alt_data, forcings_fields_data, frcing); + }); + + std::vector frcing_out(pver * extcnt, 0.0); + mam4::validation::convert_2d_view_device_to_1d_vector(frcing, frcing_out); + + output.set("frcing", frcing_out); + }); +} \ No newline at end of file diff --git a/src/validation/mo_setext/setext_driver.cpp b/src/validation/mo_setext/setext_driver.cpp new file mode 100644 index 000000000..c20e82777 --- /dev/null +++ b/src/validation/mo_setext/setext_driver.cpp @@ -0,0 +1,68 @@ +// mam4xx: Copyright (c) 2022, +// Battelle Memorial Institute and +// National Technology & Engineering Solutions of Sandia, LLC (NTESS) +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include +#include + +void usage() { + std::cerr << "setext_driver: a Skywalker driver for validating the " + "MAM4 rename parameterizations." + << std::endl; + std::cerr << "setext_driver: usage:" << std::endl; + std::cerr << "setext_driver " << std::endl; + exit(0); +} + +using namespace skywalker; +using namespace mam4; + +// Parameterizations used by the setext() process. +void extfrc_set(Ensemble *ensemble); + +int main(int argc, char **argv) { + if (argc == 1) { + usage(); + } + validation::initialize(argc, argv); + std::string input_file = argv[1]; + std::string output_file = validation::output_name(input_file); + std::cout << argv[0] << ": reading " << input_file << std::endl; + + // Load the ensemble. Any error encountered is fatal. + Ensemble *ensemble = skywalker::load_ensemble(input_file, "mam4xx"); + + // the settings. + Settings settings = ensemble->settings(); + if (!settings.has("function")) { + std::cerr << "No function specified in mam4xx.settings!" << std::endl; + exit(1); + } + + // Dispatch to the requested function. + auto func_name = settings.get("function"); + try { + if (func_name == "extfrc_set") { + extfrc_set(ensemble); + } else { + std::cerr << "Error: Function name '" << func_name + << "' does not have an implemented test!" << std::endl; + exit(1); + } + + } catch (std::exception &e) { + std::cerr << argv[0] << ": Error: " << e.what() << std::endl; + } + + // Write out a Python module. + std::cout << argv[0] << ": writing " << output_file << std::endl; + ensemble->write(output_file); + + // Clean up. + delete ensemble; + validation::finalize(); +} From 9ab86a6819de56349ce3fef149a315c285f18c2e Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 20:51:35 -0600 Subject: [PATCH 06/10] mo_setext - use struct --- src/mam4xx/mo_setext.hpp | 53 ++++++++++++++++++++++ src/validation/mo_setext/extfrc_set.cpp | 58 +++++++++++++++---------- 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index 471cf0ac3..dc5253b8c 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -18,6 +18,59 @@ constexpr int extfrc_cnt = 9; // FIXME check if this constant is defined somewhere else. constexpr int extcnt = 9; //, & ! number of species with external forcing + +struct forcing{ +int frc_ndx; +bool file_alt_data; +View2D fields_data; +int nsectors; +}; + +KOKKOS_INLINE_FUNCTION +void extfrc_set2(const forcing* forcings, + const View2D &frcing) { + +#if 1 + constexpr Real zero = 0.0; + + // frcing(:,:,:) = zero; + if (extfrc_cnt < 1 || extcnt < 1) { + return; + } + + /*-------------------------------------------------------- + ! ... set non-zero forcings + !--------------------------------------------------------*/ + + for (int mm = 0; mm < extfrc_cnt; ++mm) { + // Fortran to C++ indexing + auto forcing_mm = forcings[mm]; + const int nn = forcing_mm.frc_ndx - 1; + printf("nn %d \n", nn); + for (int kk = 0; kk < pver; ++kk) { + frcing(kk, nn) = zero; + } // k + + for (int isec = 0; isec < forcing_mm.nsectors; ++isec) { + if (forcing_mm.file_alt_data) { + for (int kk = 0; kk < pver; ++kk) { + // frcing(:ncol,:,nn) = frcing(:ncol,:,nn) + & + // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) + frcing(kk, nn) += forcing_mm.fields_data(isec,pver - 1 - kk); + } // kk + } else { + // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) + for (int kk = 0; kk < pver; ++kk) { + frcing(kk, nn) += forcing_mm.fields_data(isec,kk); + } + } + } // isec + + } // end mm +#endif + + }// extfrc_set2 + KOKKOS_INLINE_FUNCTION void extfrc_set(const int *forcings_frc_ndx, const int *forcings_nsectors, const bool *forcings_file_alt_data, diff --git a/src/validation/mo_setext/extfrc_set.cpp b/src/validation/mo_setext/extfrc_set.cpp index 28956d1ab..bb2f09458 100644 --- a/src/validation/mo_setext/extfrc_set.cpp +++ b/src/validation/mo_setext/extfrc_set.cpp @@ -16,47 +16,59 @@ using namespace mo_setext; void extfrc_set(Ensemble *ensemble) { ensemble->process([=](const Input &input, Output &output) { using View1DHost = typename HostType::view_1d; + // using View2DHost = typename HostType::view_2d; - int forcings_frc_ndx[extfrc_cnt] = {}; - int forcings_nsectors[extfrc_cnt] = {}; + + forcing forcings[extfrc_cnt]; + + // int forcings_frc_ndx[extfrc_cnt] = {}; + // int forcings_nsectors[extfrc_cnt] = {}; // std::vector> data; - bool forcings_file_alt_data[extfrc_cnt] = {}; - View1DHost forcings_fields_data_host; - View1D forcings_fields_data[extfrc_cnt][4]; + // bool forcings_file_alt_data[extfrc_cnt] = {}; + // View1DHost forcings_fields_data_host; + // View1D forcings_fields_data[extfrc_cnt][4]; for (int i = 1; i <= extfrc_cnt; ++i) { - forcings_frc_ndx[i - 1] = - int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); - forcings_nsectors[i - 1] = - int(input.get_array("forcings" + std::to_string(i) + "_nsectors")[0]); - const int file_alt_data = int(input.get_array( - "forcings" + std::to_string(i) + "_file_alt_data")[0]); - forcings_file_alt_data[i - 1] = file_alt_data; - for (int isec = 1; isec <= forcings_nsectors[i - 1]; ++isec) { + + forcing forcing_mm; + + forcing_mm.frc_ndx = int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); + forcing_mm.nsectors = int(input.get_array("forcings" + std::to_string(i) + "_nsectors")[0]); + forcing_mm.file_alt_data = int(input.get_array("forcings" + std::to_string(i) + "_file_alt_data")[0]); + + forcing_mm.fields_data = View2D("data", forcing_mm.nsectors, pver); + + for (int isec = 1; isec <= forcing_mm.nsectors; ++isec) { auto label = "forcings" + std::to_string(i) + "_fields" + std::to_string(isec) + "_data"; const auto data1 = input.get_array(label); - forcings_fields_data_host = - View1DHost((Real *)data1.data(), data1.size()); - forcings_fields_data[i - 1][isec - 1] = - View1D("forcings_fields_data", data1.size()); + View1DHost forcings_fields_data_host = + View1DHost((Real *)data1.data(), data1.size()); + + const View1D data_isec = Kokkos::subview(forcing_mm.fields_data , isec-1, Kokkos::ALL()); + Kokkos::deep_copy(data_isec, forcings_fields_data_host); + } // isec + + forcings[i-1] = forcing_mm; - Kokkos::deep_copy(forcings_fields_data[i - 1][isec - 1], - forcings_fields_data_host); - } } + + View2D frcing("frcing", pver, extcnt); const int ncol = 1; auto team_policy = ThreadTeamPolicy(ncol, 1u); + + Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const ThreadTeam &team) { - extfrc_set(forcings_frc_ndx, forcings_nsectors, - forcings_file_alt_data, forcings_fields_data, frcing); - }); + // extfrc_set(forcings_frc_ndx, forcings_nsectors, + // forcings_file_alt_data, forcings_fields_data, frcing); + extfrc_set2(forcings, frcing); + }); std::vector frcing_out(pver * extcnt, 0.0); mam4::validation::convert_2d_view_device_to_1d_vector(frcing, frcing_out); From fa0f11751c8880c17095e0dda0c594a879646ba5 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 30 Oct 2023 20:58:00 -0600 Subject: [PATCH 07/10] mo_setext - clang format --- src/mam4xx/mo_setext.hpp | 30 +++++++++++-------------- src/validation/mo_setext/extfrc_set.cpp | 27 +++++++++++----------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index dc5253b8c..728ba02ce 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -18,19 +18,16 @@ constexpr int extfrc_cnt = 9; // FIXME check if this constant is defined somewhere else. constexpr int extcnt = 9; //, & ! number of species with external forcing - -struct forcing{ -int frc_ndx; -bool file_alt_data; -View2D fields_data; -int nsectors; -}; +struct forcing { + int frc_ndx; + bool file_alt_data; + View2D fields_data; + int nsectors; +}; KOKKOS_INLINE_FUNCTION -void extfrc_set2(const forcing* forcings, - const View2D &frcing) { +void extfrc_set2(const forcing *forcings, const View2D &frcing) { -#if 1 constexpr Real zero = 0.0; // frcing(:,:,:) = zero; @@ -44,7 +41,7 @@ void extfrc_set2(const forcing* forcings, for (int mm = 0; mm < extfrc_cnt; ++mm) { // Fortran to C++ indexing - auto forcing_mm = forcings[mm]; + auto forcing_mm = forcings[mm]; const int nn = forcing_mm.frc_ndx - 1; printf("nn %d \n", nn); for (int kk = 0; kk < pver; ++kk) { @@ -56,21 +53,20 @@ void extfrc_set2(const forcing* forcings, for (int kk = 0; kk < pver; ++kk) { // frcing(:ncol,:,nn) = frcing(:ncol,:,nn) + & // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) - frcing(kk, nn) += forcing_mm.fields_data(isec,pver - 1 - kk); + frcing(kk, nn) += forcing_mm.fields_data(isec, pver - 1 - kk); } // kk } else { // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) for (int kk = 0; kk < pver; ++kk) { - frcing(kk, nn) += forcing_mm.fields_data(isec,kk); + frcing(kk, nn) += forcing_mm.fields_data(isec, kk); } } } // isec - } // end mm -#endif + } // end mm + +} // extfrc_set2 - }// extfrc_set2 - KOKKOS_INLINE_FUNCTION void extfrc_set(const int *forcings_frc_ndx, const int *forcings_nsectors, const bool *forcings_file_alt_data, diff --git a/src/validation/mo_setext/extfrc_set.cpp b/src/validation/mo_setext/extfrc_set.cpp index bb2f09458..0d753fbb4 100644 --- a/src/validation/mo_setext/extfrc_set.cpp +++ b/src/validation/mo_setext/extfrc_set.cpp @@ -18,8 +18,7 @@ void extfrc_set(Ensemble *ensemble) { using View1DHost = typename HostType::view_1d; // using View2DHost = typename HostType::view_2d; - - forcing forcings[extfrc_cnt]; + forcing forcings[extfrc_cnt]; // int forcings_frc_ndx[extfrc_cnt] = {}; // int forcings_nsectors[extfrc_cnt] = {}; @@ -30,11 +29,14 @@ void extfrc_set(Ensemble *ensemble) { // View1D forcings_fields_data[extfrc_cnt][4]; for (int i = 1; i <= extfrc_cnt; ++i) { - forcing forcing_mm; + forcing forcing_mm; - forcing_mm.frc_ndx = int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); - forcing_mm.nsectors = int(input.get_array("forcings" + std::to_string(i) + "_nsectors")[0]); - forcing_mm.file_alt_data = int(input.get_array("forcings" + std::to_string(i) + "_file_alt_data")[0]); + forcing_mm.frc_ndx = + int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); + forcing_mm.nsectors = + int(input.get_array("forcings" + std::to_string(i) + "_nsectors")[0]); + forcing_mm.file_alt_data = int(input.get_array( + "forcings" + std::to_string(i) + "_file_alt_data")[0]); forcing_mm.fields_data = View2D("data", forcing_mm.nsectors, pver); @@ -45,29 +47,26 @@ void extfrc_set(Ensemble *ensemble) { const auto data1 = input.get_array(label); View1DHost forcings_fields_data_host = - View1DHost((Real *)data1.data(), data1.size()); + View1DHost((Real *)data1.data(), data1.size()); - const View1D data_isec = Kokkos::subview(forcing_mm.fields_data , isec-1, Kokkos::ALL()); + const View1D data_isec = + Kokkos::subview(forcing_mm.fields_data, isec - 1, Kokkos::ALL()); Kokkos::deep_copy(data_isec, forcings_fields_data_host); } // isec - forcings[i-1] = forcing_mm; - + forcings[i - 1] = forcing_mm; } - - View2D frcing("frcing", pver, extcnt); const int ncol = 1; auto team_policy = ThreadTeamPolicy(ncol, 1u); - Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const ThreadTeam &team) { // extfrc_set(forcings_frc_ndx, forcings_nsectors, // forcings_file_alt_data, forcings_fields_data, frcing); - extfrc_set2(forcings, frcing); + extfrc_set2(forcings, frcing); }); std::vector frcing_out(pver * extcnt, 0.0); mam4::validation::convert_2d_view_device_to_1d_vector(frcing, frcing_out); From b8660291ea2391ee171a2fd20faa06708c4a67c5 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 31 Oct 2023 09:55:08 -0600 Subject: [PATCH 08/10] mo_setext - remove unsed code. --- src/mam4xx/mo_setext.hpp | 107 +++++++----------------- src/validation/mo_setext/extfrc_set.cpp | 18 +--- 2 files changed, 32 insertions(+), 93 deletions(-) diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index 728ba02ce..021b34019 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -13,12 +13,10 @@ using View1D = DeviceType::view_1d; using View2D = DeviceType::view_2d; using View3D = DeviceType::view_3d; constexpr int pver = mam4::nlev; -// FIXME get this constant constexpr int extfrc_cnt = 9; -// FIXME check if this constant is defined somewhere else. constexpr int extcnt = 9; //, & ! number of species with external forcing -struct forcing { +struct Forcing { int frc_ndx; bool file_alt_data; View2D fields_data; @@ -26,7 +24,15 @@ struct forcing { }; KOKKOS_INLINE_FUNCTION -void extfrc_set2(const forcing *forcings, const View2D &frcing) { +void extfrc_set(const Forcing *forcings, const View2D &frcing) { + + /*-------------------------------------------------------- + ... form the external forcing + --------------------------------------------------------*/ + // param[in] forcings(extcnt) array with a list of Forcing object. + // @param[out] frcing(ncol,pver,extcnt) insitu forcings [molec/cm^3/s] + // Note: we do not need zint to compute frcing + // const ColumnView &zint constexpr Real zero = 0.0; @@ -43,7 +49,6 @@ void extfrc_set2(const forcing *forcings, const View2D &frcing) { // Fortran to C++ indexing auto forcing_mm = forcings[mm]; const int nn = forcing_mm.frc_ndx - 1; - printf("nn %d \n", nn); for (int kk = 0; kk < pver; ++kk) { frcing(kk, nn) = zero; } // k @@ -63,59 +68,8 @@ void extfrc_set2(const forcing *forcings, const View2D &frcing) { } } // isec - } // end mm - -} // extfrc_set2 - -KOKKOS_INLINE_FUNCTION -void extfrc_set(const int *forcings_frc_ndx, const int *forcings_nsectors, - const bool *forcings_file_alt_data, - const View1D forcings_fields_data[extfrc_cnt][4], - const View2D &frcing) { - /*-------------------------------------------------------- - ! ... form the external forcing - !--------------------------------------------------------*/ - - // ncol ! columns in chunk - // lchnk ! chunk index - // zint(ncol, pverp) ! interface geopot above surface [km] - // frcing(ncol,pver,extcnt) ! insitu forcings [molec/cm^3/s] - // Note: we do not need zint to compute frcing - // const ColumnView &zint, - - constexpr Real zero = 0.0; - - // frcing(:,:,:) = zero; - if (extfrc_cnt < 1 || extcnt < 1) { - return; - } - - /*-------------------------------------------------------- - ! ... set non-zero forcings - !--------------------------------------------------------*/ - - for (int mm = 0; mm < extfrc_cnt; ++mm) { - // Fortran to C++ indexing - const int nn = forcings_frc_ndx[mm] - 1; - for (int kk = 0; kk < pver; ++kk) { - frcing(kk, nn) = zero; - } // k - - for (int isec = 0; isec < forcings_nsectors[mm]; ++isec) { - if (forcings_file_alt_data[mm]) { - for (int kk = 0; kk < pver; ++kk) { - // frcing(:ncol,:,nn) = frcing(:ncol,:,nn) + & - // forcings(mm)%fields(isec)%data(:ncol,pver:1:-1,lchnk) - frcing(kk, nn) += forcings_fields_data[mm][isec](pver - 1 - kk); - } // kk - } else { - // // forcings(mm)%fields(isec)%data(:ncol,:,lchnk) - for (int kk = 0; kk < pver; ++kk) { - frcing(kk, nn) += forcings_fields_data[mm][isec](kk); - } - } - } // isec - + // I did not include variables that are involved in the outfld. + // // xfcname = trim(forcings(mm)%species)//'_XFRC' // call outfld( xfcname, frcing(:ncol,:,nn), ncol, lchnk ) @@ -128,24 +82,22 @@ void extfrc_set(const int *forcings_frc_ndx, const int *forcings_nsectors, // call outfld( xfcname, frcing_col(:ncol), ncol, lchnk ) } // end mm + } // extfrc_set KOKKOS_INLINE_FUNCTION -void setext(const int *forcings_frc_ndx, const int *forcings_nsectors, - const bool *forcings_file_alt_data, - const View1D forcings_fields_data[extfrc_cnt][4], +void setext(const Forcing *forcings, const View2D &extfrc) // ! out { - /*-------------------------------------------------------- - ! ... for this latitude slice: - ! - form the production from datasets - ! - form the nox (xnox) production from lighing - ! - form the nox (xnox) production from airplanes - ! - form the co production from airplanes - !--------------------------------------------------------*/ - - // @param[in] zint(ncol,pver+1) ! interface geopot height [km] + ... for this latitude slice: + - form the production from datasets + - form the nox (xnox) production from lighing + - form the nox (xnox) production from airplanes + - form the co production from airplanes + --------------------------------------------------------*/ + + // param[in] forcings(extcnt) array with a list of Forcing object. // @param[out] extfrc(ncol,pver,extcnt) ! the "extraneous" forcing /*-------------------------------------------------------- @@ -157,17 +109,16 @@ void setext(const int *forcings_frc_ndx, const int *forcings_nsectors, /*-------------------------------------------------------- ! ... set frcing from datasets !--------------------------------------------------------*/ - extfrc_set(forcings_frc_ndx, forcings_nsectors, forcings_file_alt_data, - forcings_fields_data, + extfrc_set(forcings, extfrc); // out /*-------------------------------------------------------- - ! ... set nox production from lighting - ! note: from ground to cloud top production is c shaped - ! - ! FORTRAN refactor: nox is not included in current MAM4 - ! the related code are removed but outfld is kept for BFB testing - !--------------------------------------------------------*/ + ... set nox production from lighting + note: from ground to cloud top production is c shaped + + FORTRAN refactor: nox is not included in current MAM4 + the related code are removed but outfld is kept for BFB testing + --------------------------------------------------------*/ // no_lgt(:,:) = 0._r8 // call outfld( 'NO_Lightning', no_lgt(:ncol,:), ncol, lchnk ) diff --git a/src/validation/mo_setext/extfrc_set.cpp b/src/validation/mo_setext/extfrc_set.cpp index 0d753fbb4..850636a17 100644 --- a/src/validation/mo_setext/extfrc_set.cpp +++ b/src/validation/mo_setext/extfrc_set.cpp @@ -16,20 +16,11 @@ using namespace mo_setext; void extfrc_set(Ensemble *ensemble) { ensemble->process([=](const Input &input, Output &output) { using View1DHost = typename HostType::view_1d; - // using View2DHost = typename HostType::view_2d; - forcing forcings[extfrc_cnt]; - - // int forcings_frc_ndx[extfrc_cnt] = {}; - // int forcings_nsectors[extfrc_cnt] = {}; - // std::vector> data; - - // bool forcings_file_alt_data[extfrc_cnt] = {}; - // View1DHost forcings_fields_data_host; - // View1D forcings_fields_data[extfrc_cnt][4]; + Forcing forcings[extfrc_cnt]; for (int i = 1; i <= extfrc_cnt; ++i) { - forcing forcing_mm; + Forcing forcing_mm; forcing_mm.frc_ndx = int(input.get_array("forcings" + std::to_string(i) + "_frc_ndx")[0]); @@ -63,10 +54,7 @@ void extfrc_set(Ensemble *ensemble) { Kokkos::parallel_for( team_policy, KOKKOS_LAMBDA(const ThreadTeam &team) { - // extfrc_set(forcings_frc_ndx, forcings_nsectors, - // forcings_file_alt_data, forcings_fields_data, frcing); - - extfrc_set2(forcings, frcing); + extfrc_set(forcings, frcing); }); std::vector frcing_out(pver * extcnt, 0.0); mam4::validation::convert_2d_view_device_to_1d_vector(frcing, frcing_out); From e0f3da112ae9a3dc5620a22716a1e911473288b1 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Tue, 31 Oct 2023 10:03:50 -0600 Subject: [PATCH 09/10] mo_set - add header file for CmakeList. --- src/mam4xx/CMakeLists.txt | 1 + src/mam4xx/mo_setext.hpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/mam4xx/CMakeLists.txt b/src/mam4xx/CMakeLists.txt index f070d04ff..5c62315aa 100644 --- a/src/mam4xx/CMakeLists.txt +++ b/src/mam4xx/CMakeLists.txt @@ -43,6 +43,7 @@ install(FILES mo_photo.hpp lin_strat_chem.hpp mo_chm_diags.hpp + mo_setext.hpp DESTINATION include/mam4xx) add_library(mam4xx aero_modes.cpp) diff --git a/src/mam4xx/mo_setext.hpp b/src/mam4xx/mo_setext.hpp index 021b34019..0a4339216 100644 --- a/src/mam4xx/mo_setext.hpp +++ b/src/mam4xx/mo_setext.hpp @@ -89,6 +89,9 @@ KOKKOS_INLINE_FUNCTION void setext(const Forcing *forcings, const View2D &extfrc) // ! out { + // Note: we do not need setext. + // Because I removed diagnostic variables(i.e., output variable in outfld), + // setext and extfrc_set are equivalent. /*-------------------------------------------------------- ... for this latitude slice: - form the production from datasets From e85d43f18e47f01396dd7f2e8902eb488f4d33f2 Mon Sep 17 00:00:00 2001 From: Oscar Diaz-Ibarra Date: Mon, 13 Nov 2023 15:15:20 -0700 Subject: [PATCH 10/10] mo_setext - switch mam_x_validation to main branch --- src/validation/mam_x_validation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validation/mam_x_validation b/src/validation/mam_x_validation index eb1901c1f..748a58a60 160000 --- a/src/validation/mam_x_validation +++ b/src/validation/mam_x_validation @@ -1 +1 @@ -Subproject commit eb1901c1fe13feff81a91a9a8e5b0e02e48aa643 +Subproject commit 748a58a60f634c5c1954dbbe2cd9c4c500f0e725