Skip to content

Commit

Permalink
Add optional flag to data_override_nml which will
Browse files Browse the repository at this point in the history 
trigger a quick check of the time axis for uniformity.
If the time axis appears to be uniform in spacing
(with appropriate units, e.g. not "months"), then approximate
and construct a time axis without reading the entire unlimited axis
which can be very time consuming for large files.
  • Loading branch information
@MJHarrison-GFDL
MJHarrison-GFDL committed Dec 14, 2023
1 parent 9cbc43a commit c4214b3
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 20 deletions.
23 changes: 13 additions & 10 deletions data_override/include/data_override.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ type data_type
type(time_type), dimension(:), pointer :: time_records => NULL()
type(time_type), dimension(:), pointer :: time_prev_records => NULL()
type(time_type), dimension(:), pointer :: time_next_records => NULL()
logical :: uniform_time_axis=.false. !< the time axis is uniformly spaced
end type data_type

!> Private type for holding various data fields for performing data overrides
Expand Down Expand Up @@ -125,7 +126,8 @@ logical :: reproduce_null_char_bug = .false.
!! the netcdf file
logical :: use_data_table_yaml = .false.

namelist /data_override_nml/ debug_data_override, grid_center_bug, reproduce_null_char_bug, use_data_table_yaml
namelist /data_override_nml/ debug_data_override, grid_center_bug, reproduce_null_char_bug, use_data_table_yaml, &
uniform_times

public :: DATA_OVERRIDE_INIT_IMPL_, DATA_OVERRIDE_UNSET_ATM_, DATA_OVERRIDE_UNSET_OCN_, &
& DATA_OVERRIDE_UNSET_LND_, DATA_OVERRIDE_UNSET_ICE_, DATA_OVERRIDE_0D_, &
Expand Down Expand Up @@ -776,7 +778,7 @@ subroutine DATA_OVERRIDE_0D_(gridname,fieldname_code,data_out,time,override,data
! record fieldname, gridname in override_array
override_array(curr_position)%fieldname = fieldname_code
override_array(curr_position)%gridname = gridname
id_time = init_external_field(filename,fieldname,verbose=.false.)
id_time = init_external_field(filename,fieldname,verbose=.false.,check_uniform_times=uniform_times)
if(id_time<0) call mpp_error(FATAL,'data_override:field not found in init_external_field 1')
override_array(curr_position)%t_index = id_time
else !curr_position >0
Expand All @@ -790,7 +792,7 @@ subroutine DATA_OVERRIDE_0D_(gridname,fieldname_code,data_out,time,override,data
if_multi1: if (multifile) then
id_time_prev = -1
if_prev1: if (trim(prevfilename) /= '') then
id_time_prev = init_external_field(prevfilename,fieldname,verbose=.false.)
id_time_prev = init_external_field(prevfilename,fieldname,verbose=.false.,check_uniform_times=uniform_times)
dims = get_external_field_size(id_time)
prev_dims = get_external_field_size(id_time_prev)
! check consistency of spatial dims
Expand All @@ -803,7 +805,7 @@ subroutine DATA_OVERRIDE_0D_(gridname,fieldname_code,data_out,time,override,data
endif if_prev1
id_time_next = -1
if_next1: if (trim(nextfilename) /= '') then
id_time_next = init_external_field(nextfilename,fieldname,verbose=.false.)
id_time_next = init_external_field(nextfilename,fieldname,verbose=.false.,check_uniform_times=uniform_times)
dims = get_external_field_size(id_time)
next_dims = get_external_field_size(id_time_next)
! check consistency of spatial dims
Expand Down Expand Up @@ -1079,7 +1081,8 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d

!--- we always only pass data on compute domain
id_time = init_external_field(filename,fieldname,domain=domain,verbose=.false., &
use_comp_domain=use_comp_domain, nwindows=nwindows, ongrid=ongrid)
use_comp_domain=use_comp_domain, nwindows=nwindows, ongrid=ongrid, &
,check_uniform_times=uniform_times)

! if using consecutive files for data_override, get time axis for previous and next files
! and check spatial dims for consistency
Expand All @@ -1088,7 +1091,7 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d
if_prev4:if (trim(prevfilename) /= '') then
id_time_prev = init_external_field(prevfilename,fieldname,domain=domain, &
verbose=.false.,use_comp_domain=use_comp_domain, &
nwindows = nwindows, ongrid=ongrid)
nwindows = nwindows, ongrid=ongrid,check_uniform_times=uniform_times)
dims = get_external_field_size(id_time)
prev_dims = get_external_field_size(id_time_prev)
! check consistency of spatial dims
Expand All @@ -1103,7 +1106,7 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d
if_next4: if (trim(nextfilename) /= '') then
id_time_next = init_external_field(nextfilename,fieldname,domain=domain, &
verbose=.false.,use_comp_domain=use_comp_domain, &
nwindows = nwindows, ongrid=ongrid)
nwindows = nwindows, ongrid=ongrid,check_uniform_times=uniform_times)
dims = get_external_field_size(id_time)
next_dims = get_external_field_size(id_time_next)
! check consistency of spatial dims
Expand All @@ -1125,7 +1128,7 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d
else !ongrid=false
id_time = init_external_field(filename,fieldname,domain=domain, axis_names=axis_names,&
axis_sizes=axis_sizes, verbose=.false.,override=.true.,use_comp_domain=use_comp_domain, &
nwindows = nwindows)
nwindows = nwindows,check_uniform_times=uniform_times)

! if using consecutive files for data_override, get time axis for previous and next files
! and check spatial dims for consistency
Expand All @@ -1134,7 +1137,7 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d
if_prev5: if (trim(prevfilename) /= '') then
id_time_prev = init_external_field(prevfilename,fieldname,domain=domain, axis_names=axis_names,&
axis_sizes=axis_sizes, verbose=.false.,override=.true.,use_comp_domain=use_comp_domain, &
nwindows = nwindows)
nwindows = nwindows,check_uniform_times=uniform_times)
prev_dims = get_external_field_size(id_time_prev)
allocate(data_table(index1)%time_prev_records(prev_dims(4)))
call get_time_axis(id_time_prev,data_table(index1)%time_prev_records)
Expand All @@ -1143,7 +1146,7 @@ subroutine DATA_OVERRIDE_3D_(gridname,fieldname_code,return_data,time,override,d
if_next5: if (trim(nextfilename) /= '') then
id_time_next = init_external_field(nextfilename,fieldname,domain=domain, axis_names=axis_names,&
axis_sizes=axis_sizes, verbose=.false.,override=.true.,use_comp_domain=use_comp_domain, &
nwindows = nwindows)
nwindows = nwindows,check_uniform_times=uniform_times)
next_dims = get_external_field_size(id_time_next)
allocate(data_table(index1)%time_next_records(next_dims(4)))
call get_time_axis(id_time_next,data_table(index1)%time_next_records)
Expand Down
39 changes: 29 additions & 10 deletions time_interp/time_interp_external2.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ end subroutine time_interp_external_init
!> @param [inout] axis_sizes array of axis lengths ordered X-Y-Z-T (optional).
function init_external_field(file,fieldname,domain,desired_units,&
verbose,axis_names, axis_sizes,override,correct_leap_year_inconsistency,&
permit_calendar_conversion,use_comp_domain,ierr, nwindows, ignore_axis_atts, ongrid )
permit_calendar_conversion,use_comp_domain,ierr, nwindows, ignore_axis_atts, ongrid,&
check_uniform_times)

character(len=*), intent(in) :: file,fieldname
logical, intent(in), optional :: verbose
Expand All @@ -283,13 +284,16 @@ function init_external_field(file,fieldname,domain,desired_units,&
integer, intent(in), optional :: nwindows
logical, optional :: ignore_axis_atts
logical, optional :: ongrid !< Optional flag indicating if the data is ongrid
logical, optional :: check_uniform_times !< Optional flag to do a minimal check
!! on a few records and assume a uniform
!! time axis if this check passes.

logical :: ongrid_local !< Flag indicating if the data is ongrid

logical :: uniform_times_check
integer :: init_external_field

real(r8_kind) :: slope, intercept
integer :: ndim,ntime,i,j
real(r8_kind) :: slope, intercept, dtime
integer :: ndim,ntime,i,j,n
integer :: iscomp,iecomp,jscomp,jecomp,isglobal,ieglobal,jsglobal,jeglobal
integer :: isdata,iedata,jsdata,jedata, dxsize, dysize,dxsize_max,dysize_max
logical :: verb, transpose_xy,use_comp_domain1
Expand Down Expand Up @@ -323,7 +327,8 @@ function init_external_field(file,fieldname,domain,desired_units,&
if (debug_this_module) verb = .true.
numwindows = 1
if(present(nwindows)) numwindows = nwindows

uniform_times_check=.false.
if (present(check_uniform_times)) uniform_times_check=check_uniform_times
units = 'same'
if (PRESENT(desired_units)) then
units = desired_units
Expand Down Expand Up @@ -384,11 +389,25 @@ function init_external_field(file,fieldname,domain,desired_units,&
allocate(pes(mpp_npes()))
call mpp_get_current_pelist(pes)
allocate(tstamp(ntime),tstart(ntime),tend(ntime),tavg(ntime))

!< Only root reads the unlimited dimension and broadcasts it to the other ranks
if (mpp_root_pe() .eq. mpp_pe()) call read_data(fileobj, timename, tstamp)
call mpp_broadcast(tstamp, size(tstamp), mpp_root_pe(), pelist=pes)
deallocate(pes)
if (check_uniform_times .and. ntime .gt. 4) then
call read_data(fileobj,timename,tstamp(,corner=(/1,/),edge_lengths=(/4,/))
if((tstamp(4)-tstamp(3)).eq.(tstamp(2)-tstamp(1))) then
call read_data(fileobj,timename,tstamp(1),unlim_dim_level=ntime)
dtime=tstamp(2)-tstamp(1)
do n=2,ntime
tstamp(n)=tstamp(n-1)+dtime
enddo
else
if (mpp_root_pe() .eq. mpp_pe()) call read_data(fileobj, timename, tstamp)
call mpp_broadcast(tstamp, size(tstamp), mpp_root_pe(), pelist=pes)
deallocate(pes)
endif
else
!< Only root reads the unlimited dimension and broadcasts it to the other ranks
if (mpp_root_pe() .eq. mpp_pe()) call read_data(fileobj, timename, tstamp)
call mpp_broadcast(tstamp, size(tstamp), mpp_root_pe(), pelist=pes)
deallocate(pes)
endif

transpose_xy = .false.
isdata=1; iedata=1; jsdata=1; jedata=1
Expand Down

0 comments on commit c4214b3

Please sign in to comment.