adding more zstd testing

CMakeLists.txt

@@ -1840,7 +1840,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s
 #####
 # Build and copy nc_test4/findplugin.sh to various places
 #####
-foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins examples/C)
+foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins h5_test examples/C)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugins/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/${CC}/findplugin.sh @ONLY NEWLINE_STYLE LF)
 endforeach()
 

configure.ac

@@ -2238,7 +2238,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)
 AC_CONFIG_FILES(s3cleanup.sh:s3cleanup.in, [chmod ugo+x s3cleanup.sh])
 AC_CONFIG_FILES(s3gc.sh:s3gc.in, [chmod ugo+x s3gc.sh])
-for FP in plugins nc_test4 nczarr_test examples/C ; do
+for FP in plugins nc_test4 nczarr_test h5_test examples/C ; do
 AC_CONFIG_FILES(${FP}/findplugin.sh:plugins/findplugin.in, [chmod ugo+x ${FP}/findplugin.sh])
 done
 AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in, [chmod ugo+x ncdap_test/findtestserver.c])

h5_test/CMakeLists.txt

@@ -24,3 +24,17 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_h_par_compress)
   add_sh_test(h5_test run_par_tests)
 ENDIF()
+
+# If zstd is enabled, run a test for it.
+if(NETCDF_ENABLE_PLUGINS)
+  if(NETCDF_ENABLE_FILTER_ZSTD)
+    build_bin_test(tst_h_zstd)
+    add_sh_test(h5_test run_h5_zstd_tests)    
+  endif()
+endif()
+
+# Copy the shell file to build directory.
+file(COPY run_h5_zstd_tests.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
+
+
+

h5_test/Makefile.am

@@ -2,16 +2,24 @@
 # Copyright 2018 University Corporation for Atmospheric Research/Unidata
 # See COPYRIGHT file for conditions of use.
 #
-# This entire directory will be skipped, unless the configure script
-# is run with --enable-netcdf-4. This directory contains tests that
-# only use HDF5; these tests don't use netCDF at all.
+# This directory contains tests that only use HDF5; these tests don't
+# use netCDF at all. These tests will only be run if netcdf-4 is
+# enabled.
 #
 # If one of these tests fails, then netCDF-4 will not work correctly.
 # Ed Hartnett, Ward Fisher
 
 # Set AM_CPPFLAGS and AM_LDFLAGS based on user choices.
 include $(top_srcdir)/lib_flags.am
 
+# Un comment to use a more verbose test driver
+# SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# TESTS_ENVIRONMENT = export SETX=1;
+
+TEST_EXTENSIONS = .sh
+
 # These programs test HDF5 to make sure it will do (some of) the stuff
 # that netCDF-4 needs.
 H5TESTS = tst_h_files tst_h_files2 tst_h_files4 tst_h_atts		\
@@ -21,7 +29,7 @@ tst_h_strings tst_h_strings1 tst_h_strings2 tst_h_ints			\
 tst_h_dimscales tst_h_dimscales1 tst_h_dimscales2 tst_h_dimscales3	\
 tst_h_enums tst_h_dimscales4 tst_h_rename
 
-# If benchmarks were turned on, build and run a bunch more tests.
+# If benchmarks were turned on, build and run more tests.
 if BUILD_BENCHMARKS
 H5TESTS += tst_h_mem
 endif # BUILD_BENCHMARKS
@@ -32,6 +40,14 @@ check_PROGRAMS = $(H5TESTS)
 # List of tests to run.
 TESTS = $(H5TESTS)
 
+# If zstd is enabled, run a test for it.
+if NETCDF_ENABLE_PLUGINS
+if HAVE_ZSTD
+check_PROGRAMS += tst_h_zstd
+TESTS += run_h5_zstd_tests.sh
+endif # HAVE_ZSTD
+endif # NETCDF_ENABLE_PLUGINS
+
 # The parallel program is run from a script.
 if TEST_PARALLEL4
 check_PROGRAMS += tst_h_par tst_h_par_compress
@@ -41,7 +57,7 @@ endif
 # We must include these files in the distribution.
 EXTRA_DIST = run_par_tests.sh.in ref_tst_h_compounds.h5		\
 ref_tst_h_compounds2.h5 run_par_tests.sh ref_tst_compounds.nc	\
-h5_err_macros.h CMakeLists.txt
+h5_err_macros.h CMakeLists.txt run_h5_zstd_tests.sh
 
 # Clean up test results.
 CLEANFILES = tst_h_*.h5

h5_test/run_h5_zstd_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright 2024 University Corporation for Atmospheric
+# Research/Unidata. See netcdf-c/COPYRIGHT file for more info.
+
+# This shell runs the zstandard tests.
+
+# Ed Hartnett
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+# Load the findplugins function
+. ${builddir}/findplugin.sh
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
+echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
+
+findplugin h5zstd
+echo ""
+echo "Testing zstandard I/O with HDF5..."
+
+./tst_h_zstd
+echo "SUCCESS!!!"
+
+exit 0

h5_test/tst_h_zstd.c

@@ -0,0 +1,185 @@
+/* This is part of the netCDF package. Copyright 2024 University
+   Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
+   conditions of use.
+
+   Test HDF5 file code. These are not intended to be exhaustive tests,
+   but they use HDF5 the same way that netCDF-4 does, so if these
+   tests don't work, than netCDF-4 won't work either.
+
+   This files tests dataset creation and writing with zstd
+   compression. This test is only run if zstd compression is enabled
+   in the build.
+
+   Ed Hartnett
+*/
+
+#include "h5_err_macros.h"
+#include <hdf5.h>
+
+#define FILE_NAME "tst_h_zstd.h5"
+#define GRP_NAME "Bebbanburg"
+#define VAR_BOOL_NAME "Uhtred"
+#define VAR_INT_NAME "Alfred"
+#define DIM1_LEN 3
+#define DIM1_LEN_20 20
+#define H5Z_FILTER_ZSTD 32015
+
+int
+main()
+{
+    hid_t fileid, grpid, spaceid, datasetid;
+    int bool_out[DIM1_LEN] = {0, 1, 0};
+    hsize_t dims[1];
+
+    printf("\n*** Checking HDF5 variable functions with zstandard compression.\n");
+    printf("*** Checking HDF5 zstd filter setting and getting...");
+#define MAX_NAME 100
+#define NUM_CD_ELEM 10
+/* HDF5 defines this... */
+#define ZSTD_NAME "zstd"
+    {
+        H5Z_filter_t filter;
+        int num_filters;
+        hid_t propid;
+        unsigned int flags, cd_values[NUM_CD_ELEM], filter_config;
+        size_t cd_nelems = NUM_CD_ELEM;
+        size_t namelen = MAX_NAME;
+        char name[MAX_NAME + 1];
+        htri_t avail = -1;
+        unsigned int id = H5Z_FILTER_ZSTD;
+        unsigned int ulevel = 1;
+        herr_t code;
+
+        /* Check that zstandard filter is available. */
+        if((avail = H5Zfilter_avail(id)) < 0) ERR;
+        if (!avail)
+        {
+            printf("zstandard filter not available, but expected to be.\n");
+            ERR;
+        }
+
+        /* Open file and create group. */
+        if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
+                                H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gcreate1(fileid, GRP_NAME, 0)) < 0) ERR;
+
+        /* Write an array of bools, with zstandard compression. */
+        dims[0] = DIM1_LEN;
+        if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
+        if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
+        if (H5Pset_chunk(propid, 1, dims)) ERR;
+        if ((code = H5Pset_filter(propid, id, H5Z_FLAG_OPTIONAL, 1, &ulevel)))
+            ERR;
+        if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
+        if ((datasetid = H5Dcreate1(grpid, VAR_BOOL_NAME, H5T_NATIVE_HBOOL,
+                                   spaceid, propid)) < 0) ERR;
+        if (H5Dwrite(datasetid, H5T_NATIVE_HBOOL, H5S_ALL, H5S_ALL, H5P_DEFAULT,
+                     bool_out) < 0) ERR;
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Sclose(spaceid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+
+        /* Now reopen the file and check. */
+        if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gopen1(fileid, GRP_NAME)) < 0) ERR;
+        if ((datasetid = H5Dopen1(grpid, VAR_BOOL_NAME)) < 0) ERR;
+        if ((propid = H5Dget_create_plist(datasetid)) < 0) ERR;
+
+        /* The possible values of filter (which is just an int) can be
+         * found in H5Zpublic.h. */
+        if ((num_filters = H5Pget_nfilters(propid)) < 0) ERR;
+        if (num_filters != 1) ERR;
+        if ((filter = H5Pget_filter2(propid, 0, &flags, &cd_nelems, cd_values,
+                                     namelen, name, &filter_config)) < 0) ERR;
+        if (filter != H5Z_FILTER_ZSTD || cd_nelems != 1 ||
+            cd_values[0] != ulevel) ERR;
+        if (strcmp(name, ZSTD_NAME)) ERR;
+
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+    }
+
+    SUMMARIZE_ERR;
+    printf("*** Checking HDF5 zstd filter with int data...");
+#define MAX_NAME 100
+#define NUM_CD_ELEM 10
+/* HDF5 defines this... */
+#define ZSTD_NAME "zstd"
+    {
+        H5Z_filter_t filter;
+        int num_filters;
+        hid_t propid;
+        unsigned int flags, cd_values[NUM_CD_ELEM], filter_config;
+        size_t cd_nelems = NUM_CD_ELEM;
+        size_t namelen = MAX_NAME;
+        char name[MAX_NAME + 1];
+        unsigned int id = H5Z_FILTER_ZSTD;
+        unsigned int ulevel = 1;
+	int int_out[DIM1_LEN_20];
+	int int_in[DIM1_LEN_20];
+        herr_t code;
+	int i;
+
+	for (i = 0; i < DIM1_LEN_20; i++)
+	    int_out[i] = i * 10;
+
+        /* Open file and create group. */
+        if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
+                                H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gcreate1(fileid, GRP_NAME, 0)) < 0) ERR;
+
+        /* Write an array of ints, with zstandard compression. */
+        dims[0] = DIM1_LEN_20;
+        if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
+        if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
+        if (H5Pset_chunk(propid, 1, dims)) ERR;
+        if ((code = H5Pset_filter(propid, id, H5Z_FLAG_OPTIONAL, 1, &ulevel)))
+            ERR;
+        if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
+        if ((datasetid = H5Dcreate1(grpid, VAR_INT_NAME, H5T_NATIVE_INT,
+                                   spaceid, propid)) < 0) ERR;
+        if (H5Dwrite(datasetid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
+                     int_out) < 0) ERR;
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Sclose(spaceid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+
+        /* Now reopen the file and check. */
+        if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gopen1(fileid, GRP_NAME)) < 0) ERR;
+        if ((datasetid = H5Dopen1(grpid, VAR_INT_NAME)) < 0) ERR;
+        if ((propid = H5Dget_create_plist(datasetid)) < 0) ERR;
+
+        /* The possible values of filter (which is just an int) can be
+         * found in H5Zpublic.h. */
+        if ((num_filters = H5Pget_nfilters(propid)) < 0) ERR;
+        if (num_filters != 1) ERR;
+        if ((filter = H5Pget_filter2(propid, 0, &flags, &cd_nelems, cd_values,
+                                     namelen, name, &filter_config)) < 0) ERR;
+        if (filter != H5Z_FILTER_ZSTD || cd_nelems != 1 ||
+            cd_values[0] != ulevel) ERR;
+        if (strcmp(name, ZSTD_NAME)) ERR;
+
+        if (H5Dread(datasetid, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
+                     int_in) < 0) ERR;
+	for (i = 0; i < DIM1_LEN_20; i++)
+	    if (int_in[i] != int_out[i]) ERR;
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+    }
+
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
+}