From 99e4250d6d2575853e630a70aaaf5bd3905f4ed4 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 27 Aug 2017 20:38:37 -0600 Subject: [PATCH] Get szip working under cmake --- CMakeLists.txt | 3 +- cf | 5 ++ cf.cmake | 6 +- nc_test4/Make0 | 9 ++- nc_test4/Makefile.am | 7 +++ nc_test4/h5testszip.c | 135 ++++++++++++++++++++++++++++++++++++++++++ nc_test4/tst_szip.c | 30 +++++++--- 7 files changed, 182 insertions(+), 13 deletions(-) create mode 100755 nc_test4/h5testszip.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ac9330af..d65f6b16a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -689,10 +689,11 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_deflate "" HAVE_H5PSET_DEFLATE) #Check to see if H5Z_SZIP exists in HDF5_Libraries. If so, we must use szip. - CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5P_SZIP "" USE_SZIP) + CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Z_SZIP "" USE_SZIP) IF(USE_SZIP) FIND_LIBRARY(SZIP NAMES szip sz) IF(SZIP) + SET(HAVE_H5Z_SZIP 1) SET(SZIP_LIBRARY ${SZIP}) SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${SZIP}) ELSE() diff --git a/cf b/cf index 37a91d18b..517eb4506 100644 --- a/cf +++ b/cf @@ -6,6 +6,7 @@ FAST=1 HDF5=1 DAP=1 +SZIP=1 #HDF4=1 #PNETCDF=1 #PAR4=1 @@ -69,6 +70,10 @@ if test "x$HDF4" = "x1" ; then LDFLAGS="$LDFLAGS -ljpeg" fi +if test "x$SZIP" = "x1" ; then +LDFLAGS="$LDFLAGS -lsz -laec" +fi + export PKG_CONFIG_PATH=/usr/lib/pkgconfig if test "x$DAP" = "x1" ; then if curl-config --version >/dev/null ; then diff --git a/cf.cmake b/cf.cmake index f0625b5b7..fdc07024e 100644 --- a/cf.cmake +++ b/cf.cmake @@ -1,5 +1,5 @@ # Visual Studio -VS=1 +#VS=1 #VSSETUP=1 #export NCPATHDEBUG=1 @@ -16,8 +16,10 @@ DAP=1 if test "x$VS" != x ; then FLAGS="-DCMAKE_PREFIX_PATH=c:/tools/nccmake" -fi FLAGS="$FLAGS -DCMAKE_INSTALL_PREFIX=d:/ignore" +else +FLAGS="$FLAGS -DCMAKE_INSTALL_PREFIX=/tmp/ignore" +fi if test "x$DAP" = x ; then FLAGS="$FLAGS -DENABLE_DAP=false" diff --git a/nc_test4/Make0 b/nc_test4/Make0 index 35d58b73c..2eec144d2 100644 --- a/nc_test4/Make0 +++ b/nc_test4/Make0 @@ -29,7 +29,14 @@ all:: cmp cmp:: export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \ + ${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS} cpp:: ${CC} -E ${CFLAGS} ${T}.c > ${T}.txt + +H5=H5szip_example +EXT=testszip.nc +hdf5:: + export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ + ${CC} -o h5 ${CFLAGS} ${H5}.c ${SRC} ${LDFLAGS}; \ + ${CMD} ./h5 ${EXT} diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 1091c2626..6179d8b1b 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -106,6 +106,13 @@ endif # USE_HDF4_FILE_TESTS #-lhdf5 -lz endif # USE_HDF4 +# Szip Tests +if USE_SZIP +check_PROGRAMS += tst_szip +TESTS += tst_szip +endif + + # This will run a bunch of the test programs with valgrind, the memory # checking tool. (Valgrind must be present for this to work.) if USE_VALGRIND_TESTS diff --git a/nc_test4/h5testszip.c b/nc_test4/h5testszip.c new file mode 100755 index 000000000..36cd6dd78 --- /dev/null +++ b/nc_test4/h5testszip.c @@ -0,0 +1,135 @@ +/* + * Example illustrates the use of SZIP compression in HDF5 + */ + +#include +#include "hdf5.h" +#define NX 500 +#define NY 600 +#define CH_NX 100 +#define CH_NY 25 + +static void initialize(); + +static float buf[NX][NY]; +static float buf_r[NX][NY]; + +int main(int argc, char** argv) +{ + hid_t file, data_space, dataset32; + hid_t properties, lcpl_id, dapl_id; + hsize_t dims[2], chunk_size[2]; + int i, j; + unsigned szip_options_mask; + unsigned szip_pixels_per_block; + const char* existingfile = NULL; + + if(argc > 1) + existingfile = argv[1]; + + initialize(); + + if(!existingfile) { + + /* Describe the size of the array. */ + dims[0] = NX; + dims[1] = NY; + data_space = H5Screate_simple (2, dims, NULL); + + /* + * Create a new file using read/write access, default file + * creation properties, and default file access properties. + */ + file = H5Fcreate ("test.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* + * Set the dataset creation property list to specify that + * the raw data is to be partitioned into 100x100 element + * chunks and that each chunk is to be compressed. + */ + chunk_size[0] = CH_NX; + chunk_size[1] = CH_NY; + properties = H5Pcreate (H5P_DATASET_CREATE); + H5Pset_chunk (properties, 2, chunk_size); + + /* + * Set parameters for SZIP compression; check the description of + * the H5Pset_szip function in the HDF5 Reference Manual for more + * information. + */ + szip_options_mask=H5_SZIP_NN_OPTION_MASK; + szip_pixels_per_block=32; + + H5Pset_szip (properties, szip_options_mask, szip_pixels_per_block); + + /* + * Create a new dataset within the file. The datatype + * and data space describe the data on disk, which may + * be different from the format used in the application's + * memory. + */ + + lcpl_id = H5Pcreate (H5P_LINK_CREATE); + dapl_id = H5Pcreate (H5P_DATASET_ACCESS); + + dataset32 = H5Dcreate (file, "datasetF32", H5T_NATIVE_FLOAT, data_space, lcpl_id, properties, dapl_id); + + /* + * Write the array to the file. The datatype and dataspace + * describe the format of the data in the `buf' buffer. + * The raw data is translated to the format required on disk, + * as defined above. We use default raw data transfer properties. + */ + + H5Dwrite (dataset32, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, buf); + + } /*!existingfile */ + + + if(existingfile) { + file = H5Fopen (existingfile, H5F_ACC_RDONLY, H5P_DEFAULT); + properties = H5Pcreate(H5P_DATASET_ACCESS); + dataset32 = H5Dopen(file, "datasetF32", properties); + } /*!existingfile */ + + /* + * Read the array. This is similar to writing data, + * except the data flows in the opposite direction. + * Note: Decompression is automatic. + */ + + H5Dread (dataset32, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, buf_r); + + /* Do comparison */ + for (i=0; i < NX; i++) { + for (j=0; j < NY; j++) { + if(buf[i][j] != buf_r[i][j]) { + printf("mismatch: [%d][%d]: write = %f read=%f\n", + i,j,buf[i][j],buf_r[i][j]); + } + } + } + + + if(!existingfile) { + H5Sclose (data_space); + H5Pclose (properties); + } + H5Dclose (dataset32); + H5Fclose (file); + return 0; +} + +static void +initialize(void) +{ + int i, j; + /* Initialize data buffer with some bogus data. */ + for(i=0; i < NX; i++) { + for(j=0; j < NY; j++) { + buf[i][j] = (float)(i + j); + } + } +} diff --git a/nc_test4/tst_szip.c b/nc_test4/tst_szip.c index 059ecdc69..623145254 100644 --- a/nc_test4/tst_szip.c +++ b/nc_test4/tst_szip.c @@ -16,15 +16,19 @@ #include "netcdf.h" #include "netcdf_filter.h" -#define PLAIN -#undef USECLOSE +#undef PLAIN +#define USECLOSE /* Szip Constants. */ #define HDF5_FILTER_SZIP 4 -#define H5_SZIP_EC_OPTION_MASK 4 -#define H5_SZIP_NN_OPTION_MASK 32 #define H5_SZIP_MAX_PIXELS_PER_BLOCK 32 +/* Option Mask Flags (Refere to HDF5 szip documentation) */ +#define H5_SZIP_ALLOW_K13_OPTION_MASK 1 /*Allows k split = 13 compression mode. (Default)*/ +#define H5_SZIP_CHIP_OPTION_MASK 2 /*Compresses exactly as in hardware*/ +#define H5_SZIP_EC_OPTION_MASK 4 /*Selects entropy coding method. (Default)*/ +#define H5_SZIP_NN_OPTION_MASK 32 /*Selects nearest neighbor coding method*/ + #define NX 500 #define NY 600 #define CH_NX 100 @@ -50,7 +54,10 @@ main(void) if(nc_def_dim(ncid, "y", dims[1], &dimids[1])) ERR; /* Create a dimensioned variable */ - if(nc_def_var(ncid, "var", NC_FLOAT, 2, dimids, &varid)) ERR; + if(nc_def_var(ncid, "datasetF32", NC_FLOAT, 2, dimids, &varid)) ERR; + + /* no fill */ + if(nc_def_var_fill(ncid, varid, 1, NULL)) ERR; /* Define chunking for the variable: * the raw data is to be partitioned into 100x100 element chunks. @@ -67,7 +74,12 @@ main(void) */ szip_params[0] = H5_SZIP_NN_OPTION_MASK; szip_params[1] = H5_SZIP_MAX_PIXELS_PER_BLOCK; - if(nc_def_var_filter(ncid, varid, HDF5_FILTER_SZIP, 2, szip_params)) ERR; + { int stat = nc_def_var_filter(ncid, varid, HDF5_FILTER_SZIP, 2, szip_params); + if(stat) { + fprintf(stderr,"XXX: %d %s\b",stat,nc_strerror(stat)); + ERR; + } + } #endif if(nc_enddef(ncid)) ERR; @@ -82,11 +94,11 @@ main(void) /* Write the array to the file */ if(nc_put_var_float(ncid, varid, &buf[0][0])) ERR; -#if USECLOSE +#ifdef USECLOSE /* Close and re-open the file */ if(nc_close(ncid)) ERR; - if(nc_open("testszip.nc", NC_NETCDF4. &ncid)) ERR; - if(nc_inq_varid(ncid, "var", &varid)) ERR; + if(nc_open("testszip.nc", NC_NETCDF4, &ncid)) ERR; + if(nc_inq_varid(ncid, "datasetF32", &varid)) ERR; #endif /*