merge of latest dev work from GFDL Weather and Climate Dynamics Divis…

…ion (#114) * merge of latest dev work from GFDL Weather and Climate Dynamics Division * remove trailing whitespace from source files * release notes updates in RELEASE.md * Remove blocks of debug code and commented code
NOAA-GFDL · Jul 8, 2021 · 77e92f6 · 77e92f6
1 parent 2f53d89
commit 77e92f6
Show file tree

Hide file tree

Showing 21 changed files with 3,559 additions and 6,294 deletions.
diff --git a/GFDL_tools/fv_diag_column.F90 b/GFDL_tools/fv_diag_column.F90
@@ -381,7 +381,7 @@ subroutine debug_column(pt, delp, delz, u, v, w, q, npz, ncnst, sphum, nwat, zvi
        write(unit, *) '==================================================================='
        write(unit, *)
        flush(unit)
- 
+
     enddo
 
   end subroutine debug_column

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,3 +1,23 @@
+
+# RELEASE NOTES for FV3 202107: Summary
+
+FV3-202107-public --- 08 July 2021
+Lucas Harris, GFDL [email protected]
+
+This version has been tested against the current SHiELD physics
+and with FMS release 2021.02 from https://github.com/NOAA-GFDL/FMS
+
+This release includes the following:
+
+- Comprehensive documentation in LaTEX format (FV3 team)
+- Default changes to some namelist options and updated inline documentation
+- Multiple same-level and telescoping nests for the Regional domain (J Mouallem)
+- Updated fms2_io functionality (L Chilutti)
+- Revised Regional domain code (K-Y Cheng)
+- Reproducibility fixes for global+nests and regional+nests (tested for absolute reproducibility across PE counts, restarts)
+- Other updates and general cleanup
+
+
 # RELEASE NOTES for FV3 202101: Summary
 
 FV3-202101-public --- 22 January 2021
@@ -12,8 +32,8 @@ This release includes the following:
 - In-line GFDL Microphysics
 - Fast-timescale Rayleigh damping
 - Updated namelist documentation
-- Implemented multiple same-level and telescoping nests for the global system (from J Mouallem)
-- Updated coarse-graining capabilities (from S Clark)
+- Implemented multiple same-level and telescoping nests for the global system (J Mouallem)
+- Updated coarse-graining capabilities (S Clark)
 - Re-organized fv_diagnostics, moving the revised fv_diag_column functionality and the declaration of diagnostic IDs to separate files
 - and other updates and general cleanup
 
@@ -23,6 +43,7 @@ This version of FV3 is described as component of SHiELD in Harris et al. (2020,
 
 atmosphere.F90: if using the in-line GFDL microphysics the precipitation rates (available in the structure Atm%inline_mp for rain, ice, snow, and graupel separately) must be passed into the physics and/or land model as appropriate. Here we demonstrate how to do this in SHiELD by copying them into IPD_Data(nb)%Statein%prep (and so on), which are newly defined in the IPD_Data structure within the SHiELD physics.
 
+
 # RELEASE NOTES for FV3 201912: Summary
 
 FV3-201912-public --- 10 January 2020
@@ -33,12 +54,12 @@ and with FMS release candidate 2020.02 from https://github.com/NOAA-GFDL/FMS
 
 Includes all of the features of the GFDL Release to EMC, as well as:
 
-- Updated 2017 GFDL Microphysics (from S-J Lin and L Zhou included in GFSv15)
-- Updates for GFSv15 ICs (from T Black/J Abeles, EMC)
-- Updates to support new nesting capabilities in FMS (from Z Liang)
+- Updated 2017 GFDL Microphysics (S-J Lin and L Zhou included in GFSv15)
+- Updates for GFSv15 ICs (T Black/J Abeles, EMC)
+- Updates to support new nesting capabilities in FMS (Z Liang)
 - Re-written grid nesting code for efficiency and parallelization
 - Re-organized fv_eta for improved vertical level selection
-- 2018 Stand-alone regional capabilities (from T Black/J Abeles, EMC)
+- 2018 Stand-alone regional capabilities (T Black/J Abeles, EMC)
 - Refactored model front-end (fv_control, fv_restart)
 - Support for point soundings
 - And other updates

diff --git a/model/a2b_edge.F90 b/model/a2b_edge.F90
@@ -463,175 +463,4 @@ real function extrap_corner ( p0, p1, p2, q1, q2 )
 
   end function extrap_corner
 
-!!$#ifdef TEST_VAND2
-!!$  subroutine a2b_ord4(qin, qout, grid, agrid, npx, npy, is, ie, js, je, ng, replace)
-!!$! use  tp_core_mod,      only: copy_corners
-!!$  integer, intent(IN):: npx, npy, is, ie, js, je, ng
-!!$  real, intent(INOUT)::  qin(is-ng:ie+ng,js-ng:je+ng)   ! A-grid field
-!!$  real, intent(INOUT):: qout(is-ng:ie+ng,js-ng:je+ng)   ! Output  B-grid field
-!!$  real,    intent(in) ::  grid(is-ng:ie+ng+1,js-ng:je+ng+1,2)
-!!$  real,    intent(in) :: agrid(is-ng:ie+ng,js-ng:je+ng,2)
-!!$  logical, optional, intent(IN):: replace
-!!$  real qx(is:ie+1,js-ng:je+ng)
-!!$  real qy(is-ng:ie+ng,js:je+1)
-!!$  real:: p0(2)
-!!$  integer :: i, j
-!!$
-!!$  real, pointer, dimension(:,:,:) :: grid, agrid
-!!$  real, pointer, dimension(:,:)   :: dxa, dya
-!!$
-!!$  real, pointer, dimension(:) :: edge_w, edge_e, edge_s, edge_n
-!!$
-!!$  edge_w => gridstruct%edge_w
-!!$  edge_e => gridstruct%edge_e
-!!$  edge_s => gridstruct%edge_s
-!!$  edge_n => gridstruct%edge_n
-!!$
-!!$  grid => gridstruct%grid
-!!$  agrid => gridstruct%agrid
-!!$  dxa => gridstruct%dxa
-!!$  dya => gridstruct%dya
-!!$
-!!$
-!!$  if (gridstruct%grid_type < 3) then
-!!$
-!!$!------------------------------------------
-!!$! Copy fields to the phantom corner region:
-!!$!------------------------------------------
-!!$!  call copy_corners(qin, npx, npy, 1)
-!!$
-!!$  do j=js,je+1
-!!$     do i=is,ie+1
-!!$!SW:
-!!$     if ( i==1 .and. j==1 ) goto 123
-!!$          if ( i==2 .and. j==1 ) then
-!!$               qin(0,-1) = qin(-1,2)
-!!$               qin(0, 0) = qin(-1,1)
-!!$          endif
-!!$          if ( i==1 .and. j==2 ) then
-!!$               qin(-1,0) = qin(2,-1)
-!!$               qin( 0,0) = qin(1,-1)
-!!$          endif
-!!$          if ( i==2 .and. j==2 ) then
-!!$               qin( 0,0) = qin(4,4)
-!!$          endif
-!!$!SE:
-!!$      if ( i==npx   .and. j==1 ) goto 123
-!!$          if ( i==npx-1 .and. j==1 ) then
-!!$               qin(npx,-1) = qin(npx+1,2)
-!!$               qin(npx, 0) = qin(npx+1,1)
-!!$          endif
-!!$          if ( i==npx-1 .and. j==2 ) then
-!!$               qin(npx,0) = qin(npx-4,4)
-!!$          endif
-!!$          if ( i==npx   .and. j==2 ) then
-!!$               qin(npx+1,0) = qin(npx-2,-1)
-!!$               qin(npx,  0) = qin(npx-1,-1)
-!!$          endif
-!!$!NE:
-!!$      if ( i==npx   .and. j==npy   ) goto 123
-!!$          if ( i==npx-1 .and. j==npy-1 ) then
-!!$               qin(npx,npy) = qin(npx-4,npy-4)
-!!$          endif
-!!$          if ( i==npx   .and. j==npy-1 ) then
-!!$               qin(npx+1,npy) = qin(npx-2,npy+1)
-!!$               qin(npx,  npy) = qin(npx-1,npy+1)
-!!$          endif
-!!$          if ( i==npx-1 .and. j==npy   ) then
-!!$               qin(npx,npy+1) = qin(npx+1,npy-2)
-!!$               qin(npx,npy  ) = qin(npx+1,npy-1)
-!!$          endif
-!!$!NW:
-!!$      if ( i==1 .and. j==npy   ) goto 123
-!!$          if ( i==1 .and. j==npy-1 ) then
-!!$               qin(-1,npy) = qin(2,npy+1)
-!!$               qin( 0,npy) = qin(1,npy+1)
-!!$          endif
-!!$          if ( i==2 .and. j==npy-1 ) then
-!!$               qin(0,npy) = qin(4,npy-4)
-!!$          endif
-!!$          if ( i==2 .and. j==npy   ) then
-!!$               qin(0,npy+1) = qin(-1,npy-2)
-!!$               qin(0,npy  ) = qin(-1,npy-1)
-!!$          endif
-!!$
-!!$          qout(i,j) = van2(1, i,j)*qin(i-2,j-2) + van2(2, i,j)*qin(i-1,j-2) +  &
-!!$                      van2(3, i,j)*qin(i  ,j-2) + van2(4, i,j)*qin(i+1,j-2) +  &
-!!$                      van2(5, i,j)*qin(i-2,j-1) + van2(6, i,j)*qin(i-1,j-1) +  &
-!!$                      van2(7, i,j)*qin(i  ,j-1) + van2(8, i,j)*qin(i+1,j-1) +  &
-!!$                      van2(9, i,j)*qin(i-2,j  ) + van2(10,i,j)*qin(i-1,j  ) +  &
-!!$                      van2(11,i,j)*qin(i  ,j  ) + van2(12,i,j)*qin(i+1,j  ) +  &
-!!$                      van2(13,i,j)*qin(i-2,j+1) + van2(14,i,j)*qin(i-1,j+1) +  &
-!!$                      van2(15,i,j)*qin(i  ,j+1) + van2(16,i,j)*qin(i+1,j+1)
-!!$123  continue
-!!$     enddo
-!!$  enddo
-!!$
-!!$! 3-way extrapolation
-!!$    if ( gridstruct%sw_corner ) then
-!!$          p0(1:2) = grid(1,1,1:2)
-!!$        qout(1,1) = (extrap_corner(p0, agrid(1,1,1:2), agrid( 2, 2,1:2), qin(1,1), qin( 2, 2)) + &
-!!$                     extrap_corner(p0, agrid(0,1,1:2), agrid(-1, 2,1:2), qin(0,1), qin(-1, 2)) + &
-!!$                     extrap_corner(p0, agrid(1,0,1:2), agrid( 2,-1,1:2), qin(1,0), qin( 2,-1)))*r3
-!!$
-!!$    endif
-!!$    if ( gridstruct%se_corner ) then
-!!$            p0(1:2) = grid(npx,1,1:2)
-!!$        qout(npx,1) = (extrap_corner(p0, agrid(npx-1,1,1:2), agrid(npx-2, 2,1:2), qin(npx-1,1), qin(npx-2, 2)) + &
-!!$                       extrap_corner(p0, agrid(npx-1,0,1:2), agrid(npx-2,-1,1:2), qin(npx-1,0), qin(npx-2,-1)) + &
-!!$                       extrap_corner(p0, agrid(npx  ,1,1:2), agrid(npx+1, 2,1:2), qin(npx  ,1), qin(npx+1, 2)))*r3
-!!$    endif
-!!$    if ( gridstruct%ne_corner ) then
-!!$              p0(1:2) = grid(npx,npy,1:2)
-!!$        qout(npx,npy) = (extrap_corner(p0, agrid(npx-1,npy-1,1:2), agrid(npx-2,npy-2,1:2), qin(npx-1,npy-1), qin(npx-2,npy-2)) + &
-!!$                         extrap_corner(p0, agrid(npx  ,npy-1,1:2), agrid(npx+1,npy-2,1:2), qin(npx  ,npy-1), qin(npx+1,npy-2)) + &
-!!$                         extrap_corner(p0, agrid(npx-1,npy  ,1:2), agrid(npx-2,npy+1,1:2), qin(npx-1,npy  ), qin(npx-2,npy+1)))*r3
-!!$    endif
-!!$    if ( gridstruct%nw_corner ) then
-!!$            p0(1:2) = grid(1,npy,1:2)
-!!$        qout(1,npy) = (extrap_corner(p0, agrid(1,npy-1,1:2), agrid( 2,npy-2,1:2), qin(1,npy-1), qin( 2,npy-2)) + &
-!!$                       extrap_corner(p0, agrid(0,npy-1,1:2), agrid(-1,npy-2,1:2), qin(0,npy-1), qin(-1,npy-2)) + &
-!!$                       extrap_corner(p0, agrid(1,npy,  1:2), agrid( 2,npy+1,1:2), qin(1,npy  ), qin( 2,npy+1)))*r3
-!!$    endif
-!!$
-!!$ else  ! grid_type>=3
-!!$
-!!$!------------------------
-!!$! Doubly periodic domain:
-!!$!------------------------
-!!$! X-sweep: PPM
-!!$    do j=js-2,je+2
-!!$       do i=is,ie+1
-!!$          qx(i,j) = b1*(qin(i-1,j)+qin(i,j)) + b2*(qin(i-2,j)+qin(i+1,j))
-!!$       enddo
-!!$    enddo
-!!$! Y-sweep: PPM
-!!$    do j=js,je+1
-!!$       do i=is-2,ie+2
-!!$          qy(i,j) = b1*(qin(i,j-1)+qin(i,j)) + b2*(qin(i,j-2)+qin(i,j+1))
-!!$       enddo
-!!$    enddo
-!!$
-!!$    do j=js,je+1
-!!$       do i=is,ie+1
-!!$          qout(i,j) = 0.5*( a1*(qx(i,j-1)+qx(i,j  ) + qy(i-1,j)+qy(i,  j)) +  &
-!!$                            a2*(qx(i,j-2)+qx(i,j+1) + qy(i-2,j)+qy(i+1,j)) )
-!!$       enddo
-!!$    enddo
-!!$
-!!$ endif
-!!$
-!!$ if ( present(replace) ) then
-!!$     if ( replace ) then
-!!$          do j=js,je+1
-!!$             do i=is,ie+1
-!!$                qin(i,j) = qout(i,j)
-!!$             enddo
-!!$          enddo
-!!$     endif
-!!$ endif
-!!$
-!!$  end subroutine a2b_ord4
-!!$#endif
-
 end module a2b_edge_mod
diff --git a/model/boundary.F90 b/model/boundary.F90
@@ -542,35 +542,6 @@ subroutine fill_nested_grid_3D(var_nest, var_coarse, ind, wt, istag, jstag,  &
 
  end subroutine fill_nested_grid_3D
 
-!!$ subroutine nested_grid_BC_mpp_2d(var_nest, nest_domain, ind, wt, istag, jstag, &
-!!$      npx, npy, bd, isg, ieg, jsg, jeg, nstep_in, nsplit_in, proc_in)
-!!$
-!!$   type(fv_grid_bounds_type), intent(IN) :: bd
-!!$   real, dimension(bd%isd:bd%ied+istag,bd%jsd:bd%jed+jstag), intent(INOUT) :: var_nest
-!!$   real, dimension(isg:ieg+istag,jsg:jeg+jstag), intent(IN) :: var_coarse
-!!$   type(nest_domain_type), intent(INOUT) :: nest_domain
-!!$   integer, dimension(bd%isd:bd%ied+istag,bd%jsd:bd%jed+jstag,2), intent(IN) :: ind
-!!$   real, dimension(bd%isd:bd%ied+istag,bd%jsd:bd%jed+jstag,4), intent(IN) :: wt
-!!$   integer, intent(IN) :: istag, jstag, npx, npy, isg, ieg, jsg, jeg
-!!$   integer, intent(IN), OPTIONAL :: nstep_in, nsplit_in
-!!$   logical, intent(IN), OPTIONAL :: proc_in
-!!$
-!!$   real, dimension(bd%isd:bd%ied+istag,bd%jsd:bd%jed+jstag,1) :: var_nest_3d
-!!$
-!!$   integer :: i,j
-!!$
-!!$   do j=bd%jsd,bd%jed+jstag
-!!$   do i=bd%isd,bd%ied+istag
-!!$      var_nest_3d(i,j,1) = var_nest(i,j)
-!!$   enddo
-!!$   enddo
-!!$
-!!$   call nested_grid_BC_mpp_3d(var_nest_3d, nest_domain, ind, wt, istag, jstag, &
-!!$      npx, npy, 1, bd, isg, ieg, jsg, jeg, nstep_in, nsplit_in, proc_in)
-!!$
-!!$
-!!$ end subroutine nested_grid_BC_mpp_2d
-
  subroutine nested_grid_BC_mpp_3d(var_nest, var_coarse, nest_domain, ind, wt, istag, jstag, &
       npx, npy, npz, bd, isg, ieg, jsg, jeg, nest_level, nstep_in, nsplit_in, proc_in)
 
@@ -1717,11 +1688,6 @@ subroutine nested_grid_BC_recv_scalar(nest_domain, istag, jstag, npz, &
 
    integer                      :: position
 
-!!$   integer                      :: isw_f, iew_f, jsw_f, jew_f, isw_c, iew_c, jsw_c, jew_c
-!!$   integer                      :: ise_f, iee_f, jse_f, jee_f, ise_c, iee_c, jse_c, jee_c
-!!$   integer                      :: iss_f, ies_f, jss_f, jes_f, iss_c, ies_c, jss_c, jes_c
-!!$   integer                      :: isn_f, ien_f, jsn_f, jen_f, isn_c, ien_c, jsn_c, jen_c
-
    integer :: i,j, k
 
    if (istag == 1 .and. jstag == 1) then
@@ -2543,7 +2509,7 @@ end subroutine fill_var_coarse
 
   subroutine update_coarse_grid_mpp_vector(u_coarse, v_coarse, u_nest, v_nest, nest_domain, dx, dy, area, &
       bd, isd_p, ied_p, jsd_p, jed_p, is_n, ie_n, js_n, je_n, &
-      isu, ieu, jsu, jeu, npx, npy, npz, istag_u, jstag_u, istag_v, jstag_v, &
+      isu, ieu, jsu, jeu, jeu_stag, iev_stag, npx, npy, npz, istag_u, jstag_u, istag_v, jstag_v, &
       r, nestupdate, upoff, nsponge, &
       parent_proc, child_proc, parent_grid, nest_level, flags, gridtype)
 
@@ -2553,7 +2519,7 @@ subroutine update_coarse_grid_mpp_vector(u_coarse, v_coarse, u_nest, v_nest, nes
 
    type(fv_grid_bounds_type), intent(IN) :: bd
    integer, intent(IN) :: isd_p, ied_p, jsd_p, jed_p, is_n, ie_n, js_n, je_n
-   integer, intent(IN) :: isu, ieu, jsu, jeu
+   integer, intent(IN) :: isu, ieu, jsu, jeu, jeu_stag, iev_stag
    integer, intent(IN) :: istag_u, jstag_u, istag_v, jstag_v
    integer, intent(IN) :: npx, npy, npz, r, nestupdate, upoff, nsponge
    real, intent(IN)    :: u_nest(is_n:ie_n+istag_u,js_n:je_n+jstag_u,npz)
@@ -2614,13 +2580,14 @@ subroutine update_coarse_grid_mpp_vector(u_coarse, v_coarse, u_nest, v_nest, nes
    s = r/2 !rounds down (since r > 0)
    qr = r*upoff + nsponge - s
 
-   if (parent_proc .and. .not. (ieu < isu .or. jeu < jsu)) then
+   if (parent_proc .and. .not. (ieu < isu .or. jeu_stag < jsu)) then
       call fill_var_coarse(u_coarse, coarse_dat_recv_u, isd_p, ied_p, jsd_p, jed_p, &
-           isu, ieu, jsu, jeu, npx, npy, npz, istag_u, jstag_u, nestupdate, parent_grid)
+           isu, ieu, jsu, jeu_stag, npx, npy, npz, istag_u, jstag_u, nestupdate, parent_grid)
    endif
-   if (parent_proc .and. .not. (ieu < isu .or. jeu < jsu)) then
+
+   if (parent_proc .and. .not. (iev_stag < isu .or. jeu < jsu)) then
       call fill_var_coarse(v_coarse, coarse_dat_recv_v, isd_p, ied_p, jsd_p, jed_p, &
-           isu, ieu, jsu, jeu, npx, npy, npz, istag_v, jstag_v, nestupdate, parent_grid)
+           isu, iev_stag, jsu, jeu, npx, npy, npz, istag_v, jstag_v, nestupdate, parent_grid)
    endif
 
    if (allocated(coarse_dat_recv_u)) deallocate(coarse_dat_recv_u)