Merge branch 'development' into params_in_inputs

Build/cmake.sh

@@ -13,6 +13,5 @@ cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
       -DROMSX_ENABLE_TESTS:BOOL=ON \
       -DROMSX_ENABLE_FCOMPARE:BOOL=ON \
       -DROMSX_ENABLE_DOCUMENTATION:BOOL=OFF \
-      -DROMSX_ENABLE_SALINITY:BOOL=ON \
       -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
       .. && make -j8

CMake/BuildROMSXExe.cmake

@@ -20,10 +20,6 @@ function(build_romsx_lib romsx_lib_name)
                  ${ROMSX_EOS_DIR}/EOS.H)
   target_include_directories(${romsx_lib_name} SYSTEM PUBLIC ${ROMSX_EOS_DIR})
 
-  if(ROMSX_ENABLE_SALINITY)
-    target_compile_definitions(${romsx_lib_name} PUBLIC ROMSX_USE_SALINITY)
-  endif()
-
   if(ROMSX_ENABLE_PARTICLES)
     target_sources(${romsx_lib_name} PRIVATE
                    ${SRC_DIR}/Particles/TracerPC.cpp)

CMakeLists.txt

@@ -10,7 +10,6 @@ include(CMakePackageConfigHelpers)
 #General options for all executables in the project
 set(ROMSX_DIM "3" CACHE STRING "Number of physical dimensions")
 option(ROMSX_ENABLE_DOCUMENTATION "Build documentation" OFF)
-option(ROMSX_ENABLE_SALINITY      "Enable Salinity" ON)
 option(ROMSX_ENABLE_ALL_WARNINGS  "Enable all compiler warnings" OFF)
 option(ROMSX_ENABLE_TESTS         "Enable regression and unit tests" OFF)
 option(ROMSX_ENABLE_NETCDF        "Enable NetCDF IO" OFF)

Exec/CMakeLists.txt

@@ -2,4 +2,5 @@ set(romsx_lib_name romsx_srclib)
 add_library(${romsx_lib_name} OBJECT)
 include(${CMAKE_SOURCE_DIR}/CMake/BuildROMSXExe.cmake)
 build_romsx_lib(${romsx_lib_name})
+add_subdirectory(Seamount)
 add_subdirectory(Upwelling)

Exec/Make.ROMSX

@@ -73,6 +73,7 @@ endif
 
 #turn on NetCDF macro define
 ifeq ($(USE_NETCDF), TRUE)
+  USERSuffix = .NC
   DEFINES += -DROMSX_USE_NETCDF
   includes += $(shell pkg-config --cflags netcdf)
   LIBRARIES += $(shell pkg-config --libs netcdf)
@@ -89,10 +90,6 @@ ifeq ($(USE_PARTICLES), TRUE)
   DEFINES += -DROMSX_USE_PARTICLES
 endif
 
-ifeq ($(SALINITY),TRUE)
-  DEFINES += -DROMSX_USE_SALINITY
-endif
-
 CEXE_sources += AMReX_buildInfo.cpp
 CEXE_headers += $(AMREX_HOME)/Tools/C_scripts/AMReX_buildInfo.H
 INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts

Exec/ParticlesOverSeaMount/GNUmakefile

@@ -26,9 +26,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-# Physics
-SALINITY = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

Exec/ParticlesOverSeaMount/prob.cpp

@@ -72,7 +72,7 @@ init_custom_bathymetry (const Geometry& geom,
 
       Gpu::streamSynchronize();
 
-      if(!m_solverChoice.flat_bathymetry) {
+      if (!m_solverChoice.flat_bathymetry) {
           ParallelFor(makeSlab(gbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
           {
               Real val1, val2;
@@ -122,10 +122,12 @@ init_custom_prob(
         Array4<Real const> const& /*h*/,
         Array4<Real const> const& /*Zt_avg1*/,
         GeometryData const& geomdata,
-        const SolverChoice& /*m_solverChoice*/)
+        const SolverChoice& m_solverChoice)
 {
     const int khi = geomdata.Domain().bigEnd()[2];
 
+    bool l_use_salt = m_solverChoice.use_salt;
+
     AMREX_ALWAYS_ASSERT(bx.length()[2] == khi+1);
 
     ParallelFor(bx, [=, parms=parms] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
@@ -141,9 +143,9 @@ init_custom_prob(
         state(i, j, k, Temp_comp) = 1.;
 
         state(i,j,k,Temp_comp)=parms.T0+8.0*std::exp(z/50.0_rt);
-#ifdef ROMSX_USE_SALINITY
-        state(i,j,k,Salt_comp)=parms.S0;
-#endif
+        if (l_use_salt) {
+            state(i,j,k,Salt_comp)=parms.S0;
+        }
 
         // Set scalar = 0 everywhere
         state(i, j, k, Scalar_comp) = 0.0;
@@ -219,26 +221,27 @@ init_custom_vmix(const Geometry& /*geom*/, MultiFab& mf_Akv, MultiFab& mf_Akt, M
 
 void
 init_custom_hmix(const Geometry& /*geom*/, MultiFab& mf_visc2_p, MultiFab& mf_visc2_r,
-                 MultiFab& mf_diff2_salt, MultiFab& mf_diff2_temp, const SolverChoice& /*m_solverChoice*/)
+                 MultiFab& mf_diff2, const SolverChoice& /*m_solverChoice*/)
 {
     for ( MFIter mfi((mf_visc2_p), TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
       Array4<Real> const& visc2_p = (mf_visc2_p).array(mfi);
       Array4<Real> const& visc2_r = (mf_visc2_r).array(mfi);
-      Array4<Real> const& diff2_salt = (mf_diff2_salt).array(mfi);
-      Array4<Real> const& diff2_temp = (mf_diff2_temp).array(mfi);
+      Array4<Real> const& diff2   = mf_diff2.array(mfi);
       Box bx = mfi.tilebox();
       bx.grow(IntVect(NGROW,NGROW,0));
-
       Gpu::streamSynchronize();
 
-      ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
+      int ncomp = mf_diff2.nComp();
+
+      amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
       {
-          visc2_p(i,j,k) = 5.0;
-          visc2_r(i,j,k) = 5.0;
+        visc2_p(i,j,k) = 5.0;
+        visc2_r(i,j,k) = 5.0;
 
-          diff2_salt(i,j,k) = 0.0;
-          diff2_temp(i,j,k) = 0.0;
+        for (int n = 0; n < ncomp; n++) {
+            diff2(i,j,k,n) = 0.0;
+        }
       });
     }
 }

Exec/Seamount/GNUmakefile

@@ -23,9 +23,6 @@ USE_ASSERTION = TRUE
 # Debugging
 DEBUG = TRUE
 
-# Physics
-SALINITY = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

Exec/Seamount/inputs

@@ -49,7 +49,6 @@ romsx.plot_vars_1     = omega salt temp x_velocity y_velocity z_velocity
 romsx.plotfile_type   = amrex
 
 # SOLVER CHOICE
-romsx.use_gravity = true
 romsx.use_coriolis = true
 romsx.horizontal_advection_scheme = "upstream3" # upstream3 or centered4
 romsx.spatial_order = 2

Exec/Seamount/prob.cpp

@@ -29,16 +29,7 @@ init_custom_bathymetry (const Geometry& geom,
                         MultiFab& mf_Zt_avg1,
                         const SolverChoice& m_solverChoice)
 {
-    //std::unique_ptr<MultiFab>& mf_z_w = vec_z_w[lev];
-    //std::unique_ptr<MultiFab>& mf_h  = vec_hOfTheConfusingName[lev];
-    auto dx = geom.CellSizeArray();
-    auto ProbLoArr = geom.ProbLoArray();
-    auto ProbHiArr = geom.ProbHiArray();
-
     mf_h.setVal(geom.ProbHi(2));
-    Real depth = geom.ProbHi(2);
-    const int Lm = geom.Domain().size()[0];
-    const int Mm = geom.Domain().size()[1];
 
     //HACK HACK manually setting zeta to 0
     mf_Zt_avg1.setVal(0.0);
@@ -54,50 +45,33 @@ init_custom_bathymetry (const Geometry& geom,
 
       const auto & geomdata = geom.data();
 
-      int ncomp = 1;
-      //NOTE: this will need to be updated for multilevel
-      Box subdomain = geom.Domain();
-
-      int nx = subdomain.length(0);
-      int ny = subdomain.length(1);
-      int nz = subdomain.length(2);
-
-      auto N = nz; // Number of vertical "levels" aka, NZ
-      bool NSPeriodic = geomdata.isPeriodic(1);
-      bool EWPeriodic = geomdata.isPeriodic(0);
-
       amrex::Real Xsize = 320000.0_rt;
       amrex::Real Esize = 320000.0_rt;
       amrex::Real depth = 5000.0_rt;
 
-      if(!m_solverChoice.flat_bathymetry) {
-      Gpu::streamSynchronize();
-      amrex::ParallelFor(gbx2, ncomp,
-      [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
+      if(!m_solverChoice.flat_bathymetry)
       {
-          const auto prob_lo         = geomdata.ProbLo();
-          const auto dx              = geomdata.CellSize();
-
-          const Real x = prob_lo[0] + (i + 0.5) * dx[0];
-          const Real y = prob_lo[1] + (j + 0.5) * dx[1];
-
-          Real val1, val2;
-          int iFort = i+1;
-          int jFort = j+1;
-          val1 = (x-0.5_rt*Xsize)/40000.0_rt;
-          val2 = (y-0.5_rt*Esize)/40000.0_rt;
-          h(i,j,0) = depth - 4500.0_rt * std::exp(-(val1*val1+val2*val2));
-      });
+          ParallelFor(makeSlab(gbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
+          {
+              const auto prob_lo         = geomdata.ProbLo();
+              const auto dx              = geomdata.CellSize();
+
+              const Real x = prob_lo[0] + (i + 0.5) * dx[0];
+              const Real y = prob_lo[1] + (j + 0.5) * dx[1];
+
+              Real val1, val2;
+              val1 = (x-0.5_rt*Xsize)/40000.0_rt;
+              val2 = (y-0.5_rt*Esize)/40000.0_rt;
+              h(i,j,0) = depth - 4500.0_rt * std::exp(-(val1*val1+val2*val2));
+          });
+
       } else {
-      Gpu::streamSynchronize();
-      amrex::ParallelFor(gbx2, ncomp,
-      [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
-      {
-          h(i,j,0) = -geomdata.ProbLo(2);
-          if (k==0) {
+
+          ParallelFor(makeSlab(gbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
+          {
+              h(i,j,0) = -geomdata.ProbLo(2);
               h(i,j,0,1) = h(i,j,0);
-          }
-      });
+          });
       }
     }
 }
@@ -111,38 +85,33 @@ init_custom_prob(
         Array4<Real      > const& z_vel,
         Array4<Real const> const& /*z_w*/,
         Array4<Real const> const& z_r,
-        Array4<Real const> const& Hz,
-        Array4<Real const> const& h,
-        Array4<Real const> const& Zt_avg1,
+        Array4<Real const> const& /*Hz*/,
+        Array4<Real const> const& /*h*/,
+        Array4<Real const> const& /*Zt_avg1*/,
         GeometryData const& geomdata,
         const SolverChoice& m_solverChoice)
 {
-  const int khi = geomdata.Domain().bigEnd()[2];
+    bool l_use_salt = m_solverChoice.use_salt;
 
-  AMREX_ALWAYS_ASSERT(bx.length()[2] == khi+1);
+    const int khi = geomdata.Domain().bigEnd()[2];
 
-  const Real& rho_sfc   = p_0 / (R_d*m_solverChoice.T0);
-  const Real& thetabar  = m_solverChoice.T0;
-  const Real& dz        = geomdata.CellSize()[2];
-  const Real& el        = geomdata.ProbHi()[1];
-  const Real& prob_lo_z = geomdata.ProbLo()[2];
+    AMREX_ALWAYS_ASSERT(bx.length()[2] == khi+1);
 
     ParallelFor(bx, [=, parms=parms] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
     {
         // Geometry (note we must include these here to get the data on device)
-        const auto prob_lo         = geomdata.ProbLo();
-        const auto dx              = geomdata.CellSize();
+        // const auto prob_lo         = geomdata.ProbLo();
+        // const auto dx              = geomdata.CellSize();
 
-        const Real x = prob_lo[0] + (i + 0.5) * dx[0];
-        const Real y = prob_lo[1] + (j + 0.5) * dx[1];
+        // const Real x = prob_lo[0] + (i + 0.5) * dx[0];
+        // const Real y = prob_lo[1] + (j + 0.5) * dx[1];
         const Real z = z_r(i,j,k);
 
         state(i, j, k, Temp_comp) = 1.;
 
         state(i,j,k,Temp_comp)=m_solverChoice.T0+7.5_rt*std::exp(z/1000.0_rt);
-#ifdef ROMSX_USE_SALINITY
-        state(i,j,k,Salt_comp)=m_solverChoice.S0;
-#endif
+        if (l_use_salt) {
+            state(i,j,k,Salt_comp)=m_solverChoice.S0;
 
         // Set scalar = 0 everywhere
         state(i, j, k, Scalar_comp) = 0.0;
@@ -179,19 +148,20 @@ init_custom_prob(
 }
 
 void
-init_custom_vmix(const Geometry& geom, MultiFab& mf_Akv, MultiFab& mf_Akt, MultiFab& mf_z_w, const SolverChoice& m_solverChoice)
+init_custom_vmix(const Geometry& /*geom*/, MultiFab& mf_Akv, MultiFab& mf_Akt,
+                 MultiFab& /*mf_z_w*/, const SolverChoice& /*m_solverChoice*/)
 {
     for ( MFIter mfi((mf_Akv), TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
       Array4<Real> const& Akv = (mf_Akv).array(mfi);
       Array4<Real> const& Akt = (mf_Akt).array(mfi);
-      Array4<Real> const& z_w = (mf_z_w).array(mfi);
+
       Box bx = mfi.tilebox();
       bx.grow(IntVect(NGROW,NGROW,0));
-      const auto & geomdata = geom.data();
-      int ncomp = 1; Gpu::streamSynchronize();
-      amrex::ParallelFor(bx, ncomp,
-      [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
+
+      Gpu::streamSynchronize();
+
+      amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
       {
         Akv(i,j,k) = 1.0e-5_rt;
         Akt(i,j,k) = 1.0e-6_rt;
@@ -200,35 +170,36 @@ init_custom_vmix(const Geometry& geom, MultiFab& mf_Akv, MultiFab& mf_Akt, Multi
 }
 
 void
-init_custom_hmix(const Geometry& geom, MultiFab& mf_visc2_p, MultiFab& mf_visc2_r,
-    MultiFab& mf_diff2_salt, MultiFab& mf_diff2_temp, const SolverChoice& m_solverChoice)
+init_custom_hmix(const Geometry& /*geom*/, MultiFab& mf_visc2_p, MultiFab& mf_visc2_r,
+                 MultiFab& mf_diff2, const SolverChoice& /*m_solverChoice*/)
 {
     for ( MFIter mfi((mf_visc2_p), TilingIfNotGPU()); mfi.isValid(); ++mfi )
     {
-      Array4<Real> const& visc2_p = (mf_visc2_p).array(mfi);
-      Array4<Real> const& visc2_r = (mf_visc2_r).array(mfi);
-      Array4<Real> const& diff2_salt = (mf_diff2_salt).array(mfi);
-      Array4<Real> const& diff2_temp = (mf_diff2_temp).array(mfi);
+      Array4<Real> const& visc2_p_arr = mf_visc2_p.array(mfi);
+      Array4<Real> const& visc2_r_arr = mf_visc2_r.array(mfi);
+      Array4<Real> const& diff2_arr   = mf_diff2.array(mfi);
+
       Box bx = mfi.tilebox();
       bx.grow(IntVect(NGROW,NGROW,0));
-      const auto & geomdata = geom.data();
-      int ncomp = 1;
+
+      int ncomp = 2; // temperature and salt
       Gpu::streamSynchronize();
-      amrex::ParallelFor(bx, ncomp,
-      [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
+
+      amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
       {
-          visc2_p(i,j,k) = 0.0;
-          visc2_r(i,j,k) = 0.0;
+          visc2_p_arr(i,j,k) = 0.0;
+          visc2_r_arr(i,j,k) = 0.0;
 
-          diff2_salt(i,j,k) = 0.0;
-          diff2_temp(i,j,k) = 0.0;
+          for (int n = 0; n < ncomp; n++) {
+              diff2_arr(i,j,k,n) = 0.0;
+          }
       });
-    }
+    } // mfi
 }
 
 void
-init_custom_smflux(const Geometry& geom, const Real time, MultiFab& mf_sustr, MultiFab& mf_svstr,
-    const SolverChoice& m_solverChoice)
+init_custom_smflux(const Geometry& /*geom*/, const Real /*time*/,
+                   MultiFab& mf_sustr, MultiFab& mf_svstr, const SolverChoice& /*m_solverChoice*/)
 {
     mf_sustr.setVal(0.0);
     mf_svstr.setVal(0.0);

Exec/Upwelling/GNUmakefile

@@ -26,9 +26,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-# Physics
-SALINITY = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .