Merge pull request #243 from seahorce-scidac/fix_netcdf_with_cuda

add streamSynchronize after copies
seahorce-scidac · Aug 15, 2024 · fe53191 · fe53191
2 parents a02b366 + 7416277
commit fe53191
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/Source/IO/NCPlotFile.cpp b/Source/IO/NCPlotFile.cpp
@@ -364,6 +364,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
                 FArrayBox tmp_bathy;
                 tmp_bathy.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
                 tmp_bathy.template copy<RunOn::Device>((*vec_hOfTheConfusingName[lev])[mfi.index()],0,0,1);
+                Gpu::streamSynchronize();
 
                 auto nc_plot_var = ncf.var("h");
                 nc_plot_var.par_access(NC_INDEPENDENT);
@@ -375,6 +376,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_zeta;
             tmp_zeta.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp_zeta.template copy<RunOn::Device>((*vec_Zt_avg1[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("zeta");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -386,6 +388,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_temp;
             tmp_temp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp_temp.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()],Temp_comp,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("temp");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -397,6 +400,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_salt;
             tmp_salt.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp_salt.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()],Salt_comp,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("salt");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -438,6 +442,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*xvel_new[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("u");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -449,6 +454,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_ubar[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("ubar");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -459,6 +465,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_sustr[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("sustr");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -503,6 +510,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*yvel_new[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("v");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -514,6 +522,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_vbar[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("vbar");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -524,6 +533,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_svstr[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("svstr");
             nc_plot_var.par_access(NC_INDEPENDENT);