Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

parallel zstd test which works for cmake and autotools #3005

Merged
merged 5 commits into from
Sep 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1763,6 +1763,21 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
"${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
@ONLY)

if(NETCDF_ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
if(HDF5_PARALLEL)
configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_test.sh.in"
"${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
DESTINATION ${netCDF_BINARY_DIR}/nc_test4
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
configure_file("${netCDF_SOURCE_DIR}/h5_test/run_par_tests.sh.in"
"${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" @ONLY NEWLINE_STYLE LF)
file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
DESTINATION ${netCDF_BINARY_DIR}/h5_test
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
endif()
endif()

# Read in settings file, print out.
# Avoid using system-specific calls so that this
# might also work on Windows.
Expand Down
6 changes: 5 additions & 1 deletion configure.ac
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# -*- Autoconf -*-
o# -*- Autoconf -*-
## Process this file with autoconf to produce a configure script.

# This is part of Unidata's netCDF package. Copyright 2005-2018, see
Expand Down Expand Up @@ -784,6 +784,10 @@ if test "x$enable_filter_zstd" = "xyes" ; then
else
have_zstd=no
fi
AC_MSG_CHECKING([whether we are going to build with zstd])
AC_SUBST([HAS_ZSTD], [$have_zstd])
AC_MSG_RESULT([$have_zstd])

##
# End zstd checks
##
Expand Down
12 changes: 12 additions & 0 deletions nc_test4/run_par_test.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,18 @@ set -e
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh

if test "x@HAS_ZSTD@" = "xyes" ; then
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
echo "${HDF5_PLUGIN_DIR}"

# Find zstd plugin.
findplugin h5zstd
export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
fi

echo
echo "Testing MPI parallel I/O with various other mode flags..."
@MPIEXEC@ -n 1 ./tst_mode
Expand Down
54 changes: 54 additions & 0 deletions nc_test4/tst_parallel5.c
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,60 @@ main(int argc, char **argv)
if (!mpi_rank)
SUMMARIZE_ERR;
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* HAVE_H5Z_SZIP */

#ifdef HAVE_H5Z_SZIP
#ifdef HDF5_SUPPORTS_PAR_FILTERS
#define SZIP_DIM_LEN 256
#define SZIP_DIM_NAME "Barrels"
#define SZIP_VAR_NAME "Best_Sligo_Rags"
#define SZIP_PIXELS_PER_BLOCK 32
if (!mpi_rank)
printf("*** testing zstd compression with parallel I/O...");
{
int ncid, dimid, varid;
float *data;
float *data_in;
int elements_per_pe = SZIP_DIM_LEN/mpi_size;
size_t start[NDIMS1], count[NDIMS1];
int i, ret;

/* Create test data. */
if (!(data = malloc(elements_per_pe * sizeof(float)))) ERR;
for (i = 0; i < elements_per_pe; i++)
data[i] = mpi_rank + i * 0.1;

/* Crate a file with a scalar NC_BYTE value. */
if (nc_create_par(FILE, NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL,
&ncid)) ERR;
if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR;
if (nc_def_var(ncid, SZIP_VAR_NAME, NC_FLOAT, NDIMS1, &dimid, &varid)) ERR;
if ((ret = nc_def_var_zstandard(ncid, varid, 4)))
{
printf("%s\n", nc_strerror(ret));
ERR;
}
if (nc_enddef(ncid)) ERR;
start[0] = mpi_rank * elements_per_pe;
count[0] = elements_per_pe;
if (nc_put_vara_float(ncid, varid, start, count, data));
if (nc_close(ncid)) ERR;

/* Reopen the file and check. */
if (nc_open_par(FILE, 0, comm, info, &ncid)) ERR;
if (!(data_in = malloc(elements_per_pe * sizeof(float)))) ERR;
if (nc_get_vara_float(ncid, varid, start, count, data_in));
if (nc_close(ncid)) ERR;
for (i = 0; i < elements_per_pe; i++)
if (data_in[i] != data[i]) ERR;

/* Release resources. */
free(data_in);
free(data);
}
if (!mpi_rank)
SUMMARIZE_ERR;
#endif /* HDF5_SUPPORTS_PAR_FILTERS */
#endif /* HAVE_H5Z_SZIP */

/* Shut down MPI. */
Expand Down
Loading