From 7416277f8de0a85a0c091db74a76acee6829169e Mon Sep 17 00:00:00 2001
From: Ann Almgren <asalmgren@lbl.gov>
Date: Wed, 14 Aug 2024 20:07:33 -0700
Subject: [PATCH] add streamSynchronize after copies

---
 Source/IO/NCPlotFile.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Source/IO/NCPlotFile.cpp b/Source/IO/NCPlotFile.cpp
index 32ec219..40f8830 100644
--- a/Source/IO/NCPlotFile.cpp
+++ b/Source/IO/NCPlotFile.cpp
@@ -364,6 +364,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
                 FArrayBox tmp_bathy;
                 tmp_bathy.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
                 tmp_bathy.template copy<RunOn::Device>((*vec_hOfTheConfusingName[lev])[mfi.index()],0,0,1);
+                Gpu::streamSynchronize();
 
                 auto nc_plot_var = ncf.var("h");
                 nc_plot_var.par_access(NC_INDEPENDENT);
@@ -375,6 +376,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_zeta;
             tmp_zeta.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp_zeta.template copy<RunOn::Device>((*vec_Zt_avg1[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("zeta");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -386,6 +388,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_temp;
             tmp_temp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp_temp.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()],Temp_comp,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("temp");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -397,6 +400,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp_salt;
             tmp_salt.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp_salt.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()],Salt_comp,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("salt");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -438,6 +442,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*xvel_new[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("u");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -449,6 +454,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_ubar[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("ubar");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -459,6 +465,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_sustr[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("sustr");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -503,6 +510,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*yvel_new[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("v");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -514,6 +522,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_vbar[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("vbar");
             nc_plot_var.par_access(NC_INDEPENDENT);
@@ -524,6 +533,7 @@ REMORA::WriteNCPlotFile_which(int lev, int which_subdomain,
             FArrayBox tmp;
             tmp.resize(tmp_bx_2d,1,amrex::The_Pinned_Arena());
             tmp.template copy<RunOn::Device>((*vec_svstr[lev])[mfi.index()],0,0,1);
+            Gpu::streamSynchronize();
 
             auto nc_plot_var = ncf.var("svstr");
             nc_plot_var.par_access(NC_INDEPENDENT);