fix parallel read

in fact, still read only on proc 0, and broadcast what is important
seahorce-scidac · Nov 11, 2024 · f44e7c5 · f44e7c5
1 parent 868bd4b
commit f44e7c5
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 24 deletions.
diff --git a/Exec/IdealMiniGrid/inputs b/Exec/IdealMiniGrid/inputs
@@ -68,6 +68,6 @@ remora.rho0  = 1025.0  # Mean density (Kg/m3) used when Boussinesq approx is inf
 
 # These files can be found at https://github.com/seahorce-scidac/REMORA-data
 remora.ic_bc_type       = "real"
-remora.nc_init_file_0   = "idmini_ini_v1.nc"
-remora.nc_grid_file_0   = "idmini_grd_v1.nc"
-remora.nc_bdry_file     = "idmini_bry_Salt.nc"
+remora.nc_init_file_0   = "idmini_ini_v1_classic64.nc"
+remora.nc_grid_file_0   = "idmini_grd_v1_classic64.nc"
+remora.nc_bdry_file     = "idmini_bry_Salt_classic64.nc"
diff --git a/Source/IO/NCFile.H b/Source/IO/NCFile.H
@@ -109,10 +109,10 @@ void ReadNetCDFFile (const std::string& fname, amrex::Vector<std::string> names,
 {
     AMREX_ASSERT(arrays.size() == names.size());
 
+    auto ncf = ncutils::NCFile::open(fname, NC_NOCLOBBER );
+    ncmpi_begin_indep_data(ncf.ncid);
     if (amrex::ParallelDescriptor::IOProcessor())
     {
-        auto ncf = ncutils::NCFile::open(fname, NC_NOCLOBBER );
-        ncmpi_begin_indep_data(ncf.ncid);
 
         /*
         // get the dimension information
@@ -153,8 +153,9 @@ void ReadNetCDFFile (const std::string& fname, amrex::Vector<std::string> names,
 
             ncf.var(vname_to_read).get(dataPtr, start, shape);
         }
-        ncf.close();
+
     }
+    ncf.close();
 }
 
 /**
@@ -281,10 +282,8 @@ BuildFABsFromNetCDFFile (const amrex::Box& domain,
 
     amrex::Vector<NDArray<amrex::Real>> nc_arrays(nc_var_names.size());
 
-    if (amrex::ParallelDescriptor::IOProcessor())
-    {
-        ReadNetCDFFile(fname, nc_var_names, nc_arrays);
-    }
+
+    ReadNetCDFFile(fname, nc_var_names, nc_arrays); // this is filled only on proc 0
 
     for (int iv = 0; iv < nc_var_names.size(); iv++)
     {

diff --git a/Source/IO/NCFile.cpp b/Source/IO/NCFile.cpp
@@ -9,12 +9,12 @@ std::string ReadNetCDFVarAttrStr (const std::string& fname,
                                   const std::string& attr_name)
 {
     std::string attr_val;
+    auto ncf = ncutils::NCFile::open(fname, NC_NOCLOBBER);
+    ncmpi_begin_indep_data(ncf.ncid);
     if (amrex::ParallelDescriptor::IOProcessor())
     {
-        auto ncf = ncutils::NCFile::open(fname, NC_NOCLOBBER);
-        ncmpi_begin_indep_data(ncf.ncid);
         attr_val = ncf.var(var_name).get_attr(attr_name);
-        ncf.close();
     }
+    ncf.close();
     return attr_val;
 }
diff --git a/Source/IO/ReadFromBdryNetcdf.cpp b/Source/IO/ReadFromBdryNetcdf.cpp
@@ -63,21 +63,22 @@ read_bdry_from_netcdf (const Box& domain, const std::string& nc_bdry_file,
     const std::string dateTimeFormat ="%Y-%m-%d_%H:%M:%S";
 
     Real ocean_times[31];
-
+    // Check units of time stamps. Should be days.
+    std::string unit_str;
+    unit_str = ReadNetCDFVarAttrStr(nc_bdry_file, "ocean_time", "units"); // works on proc 0
     if (ParallelDescriptor::IOProcessor())
     {
-        // Check units of time stamps. Should be days.
-        std::string unit_str;
-        unit_str = ReadNetCDFVarAttrStr(nc_bdry_file, "ocean_time", "units");
         if (unit_str.find("days") == std::string::npos) {
             amrex::Print() << "Units of ocean_time given as: " << unit_str << std::endl;
             amrex::Abort("Units must be in days.");
         }
-        // Read the time stamps
-        using RARRAY = NDArray<Real>;
-        amrex::Vector<RARRAY> array_ts(1);
-        ReadNetCDFFile(nc_bdry_file, {"ocean_time"}, array_ts);
-
+    }
+    // Read the time stamps
+    using RARRAY = NDArray<Real>;
+    amrex::Vector<RARRAY> array_ts(1);
+    ReadNetCDFFile(nc_bdry_file, {"ocean_time"}, array_ts); // filled only on proc 0
+    if (ParallelDescriptor::IOProcessor())
+    {
         ntimes = array_ts[0].get_vshape()[0];
 
         // amrex::Print() << " NTIMES " << ntimes << std::endl;
@@ -135,10 +136,9 @@ read_bdry_from_netcdf (const Box& domain, const std::string& nc_bdry_file,
     // The width of the boundary region we need to read is 1
     width = 1;
 
+    ReadNetCDFFile(nc_bdry_file, nc_var_names, arrays); // does work on proc 0 only
     if (ParallelDescriptor::IOProcessor())
     {
-        ReadNetCDFFile(nc_bdry_file, nc_var_names, arrays);
-
         // Assert that the data has the same number of time snapshots
         int itimes = static_cast<int>(arrays[0].get_vshape()[0]);
         AMREX_ALWAYS_ASSERT(itimes == ntimes);