mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-12-15 07:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #662 in HDFFV/hdf5 from ~BYRN/hdf5_adb:hdf5_1_10 to hdf5_1_10

Browse Source 
* commit '10676247a825627feea046d025170b4c6849cc83':
  HDFFV-10276 Remove whitespace
  HDFFV-10276 Add UD flag option to h5repack
This commit is contained in:
Allen Byrne 2017-09-08 10:00:43 -05:00
commit 8f07727c69
9 changed files with 52 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
release_docs/INSTALL_CMake.txt
View File

@ -659,7 +659,7 @@ adding an option (${CTEST_SCRIPT_ARG}) to the platform configuration script.
############################################################################################# #############################################################################################
### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ### ### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ###
### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ### ### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ###
### ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201264 -C Release -VV -O hdf5.log ### ### ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf5.log ###
############################################################################################# #############################################################################################
cmake_minimum_required (VERSION 3.2.2 FATAL_ERROR) cmake_minimum_required (VERSION 3.2.2 FATAL_ERROR)
@ -668,13 +668,13 @@ cmake_minimum_required (VERSION 3.2.2 FATAL_ERROR)
# ctest -S HDF5config.cmake,OPTION=VALUE -C Release -VV -O test.log # ctest -S HDF5config.cmake,OPTION=VALUE -C Release -VV -O test.log
# where valid options for OPTION are: # where valid options for OPTION are:
# BUILD_GENERATOR - The cmake build generator: # BUILD_GENERATOR - The cmake build generator:
# Unix * Unix Makefiles # Unix * Unix Makefiles
# VS2017 * Visual Studio 15 2017 # VS2017 * Visual Studio 15 2017
# VS201764 * Visual Studio 15 2017 Win64 # VS201764 * Visual Studio 15 2017 Win64
# VS2015 * Visual Studio 14 2015 # VS2015 * Visual Studio 14 2015
# VS201564 * Visual Studio 14 2015 Win64 # VS201564 * Visual Studio 14 2015 Win64
# VS2013 * Visual Studio 12 2013 # VS2013 * Visual Studio 12 2013
# VS201364 * Visual Studio 12 2013 Win64 # VS201364 * Visual Studio 12 2013 Win64
# #
# INSTALLDIR - root folder where hdf5 is installed # INSTALLDIR - root folder where hdf5 is installed
# CTEST_CONFIGURATION_TYPE - Release, Debug, etc # CTEST_CONFIGURATION_TYPE - Release, Debug, etc

9
release_docs/RELEASE.txt
View File

@ -277,6 +277,15 @@ Bug Fixes since HDF5-1.10.0-patch1 release
Tools Tools
----- -----
- h5repack
h5repack always set the User Defined filter flag to H5Z_FLAG_MANDATORY.
Added another parameter to the 'UD=' option to set the flag by default
to '0' or H5Z_FLAG_MANDATORY, the other choice is '1' or H5Z_FLAG_OPTIONAL.
(ADB - 2017/08/31, HDFFV-10276)
- h5ls - h5ls
h5ls generated error on stack when it encountered a H5S_NULL h5ls generated error on stack when it encountered a H5S_NULL

1
tools/src/h5repack/h5repack.h
View File

@ -61,6 +61,7 @@ typedef struct {
typedef struct { typedef struct {
H5Z_filter_t filtn; /* filter identification number */ H5Z_filter_t filtn; /* filter identification number */
unsigned filt_flag; /* filter definition flag */
unsigned cd_values[CD_VALUES]; /* filter client data values */ unsigned cd_values[CD_VALUES]; /* filter client data values */
size_t cd_nelmts; /* filter client number of values */ size_t cd_nelmts; /* filter client number of values */
} filter_info_t; } filter_info_t;

2
tools/src/h5repack/h5repack_filters.c
View File

@ -402,7 +402,7 @@ int apply_filters(const char* name, /* object name from traverse list */
default: default:
{ {
if (H5Pset_filter(dcpl_id, obj.filter[i].filtn, if (H5Pset_filter(dcpl_id, obj.filter[i].filtn,
H5Z_FLAG_MANDATORY, obj.filter[i].cd_nelmts, obj.filter[i].filt_flag, obj.filter[i].cd_nelmts,
obj.filter[i].cd_values) < 0) obj.filter[i].cd_values) < 0)
return -1; return -1;
if (H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths) < 0) if (H5Pset_chunk(dcpl_id, obj.chunk.rank, obj.chunk.chunk_lengths) < 0)

12
tools/src/h5repack/h5repack_main.c
View File

@ -101,7 +101,7 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy\n"); PRINTVALSTREAM(rawoutstream, " -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy\n");
PRINTVALSTREAM(rawoutstream, " -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size\n"); PRINTVALSTREAM(rawoutstream, " -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " M - is an integer greater than 1, size of dataset in bytes (default is 0) \n"); PRINTVALSTREAM(rawoutstream, " M - is an integer greater than 1, size of dataset in bytes (default is 0)\n");
PRINTVALSTREAM(rawoutstream, " E - is a filename.\n"); PRINTVALSTREAM(rawoutstream, " E - is a filename.\n");
PRINTVALSTREAM(rawoutstream, " S - is an integer\n"); PRINTVALSTREAM(rawoutstream, " S - is an integer\n");
PRINTVALSTREAM(rawoutstream, " U - is a filename.\n"); PRINTVALSTREAM(rawoutstream, " U - is a filename.\n");
@ -164,8 +164,8 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " NBIT (no parameter)\n"); PRINTVALSTREAM(rawoutstream, " NBIT (no parameter)\n");
PRINTVALSTREAM(rawoutstream, " SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n"); PRINTVALSTREAM(rawoutstream, " SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n");
PRINTVALSTREAM(rawoutstream, " is either IN or DS\n"); PRINTVALSTREAM(rawoutstream, " is either IN or DS\n");
PRINTVALSTREAM(rawoutstream, " UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]>\n"); PRINTVALSTREAM(rawoutstream, " UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>\n");
PRINTVALSTREAM(rawoutstream, " required values for filter_number,cd_value_count,value_1\n"); PRINTVALSTREAM(rawoutstream, " required values for filter_number,filter_flag,cd_value_count,value_1\n");
PRINTVALSTREAM(rawoutstream, " optional values for value_2 to value_N\n"); PRINTVALSTREAM(rawoutstream, " optional values for value_2 to value_N\n");
PRINTVALSTREAM(rawoutstream, " NONE (no parameter)\n"); PRINTVALSTREAM(rawoutstream, " NONE (no parameter)\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
@ -200,16 +200,16 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " Chunked layout, with a layout size of 20x10, to objects dset1 and dset2\n"); PRINTVALSTREAM(rawoutstream, " Chunked layout, with a layout size of 20x10, to objects dset1 and dset2\n");
PRINTVALSTREAM(rawoutstream, " and remove filters to objects dset3, dset4, dset5\n"); PRINTVALSTREAM(rawoutstream, " and remove filters to objects dset3, dset4, dset5\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, "4) h5repack -L -c 10 -s 20:dtype file1 file2 \n"); PRINTVALSTREAM(rawoutstream, "4) h5repack -L -c 10 -s 20:dtype file1 file2\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " Using latest file format with maximum compact group size of 10 and\n"); PRINTVALSTREAM(rawoutstream, " Using latest file format with maximum compact group size of 10 and\n");
PRINTVALSTREAM(rawoutstream, " and minimum shared datatype size of 20\n"); PRINTVALSTREAM(rawoutstream, " and minimum shared datatype size of 20\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, "5) h5repack -f SHUF -f GZIP=1 file1 file2 \n"); PRINTVALSTREAM(rawoutstream, "5) h5repack -f SHUF -f GZIP=1 file1 file2\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " Add both filters SHUF and GZIP in this order to all datasets\n"); PRINTVALSTREAM(rawoutstream, " Add both filters SHUF and GZIP in this order to all datasets\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, "6) h5repack -f UD=307,1,9 file1 file2 \n"); PRINTVALSTREAM(rawoutstream, "6) h5repack -f UD=307,0,1,9 file1 file2\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " Add bzip2 filter to all datasets\n"); PRINTVALSTREAM(rawoutstream, " Add bzip2 filter to all datasets\n");
PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, "\n");

17
tools/src/h5repack/h5repack_parse.c
View File

@ -42,7 +42,7 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
size_t i, m, u; size_t i, m, u;
char c; char c;
size_t len = HDstrlen(str); size_t len = HDstrlen(str);
int k, l, p, q, end_obj = -1, no_param = 0; int f, k, l, p, q, end_obj = -1, no_param = 0;
unsigned j, n; unsigned j, n;
char sobj[MAX_NC_NAME]; char sobj[MAX_NC_NAME];
char scomp[10] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; char scomp[10] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
@ -209,13 +209,14 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
/*------------------------------------------------------------------------- /*-------------------------------------------------------------------------
* User Defined * User Defined
* has the format UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]> * has the format UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
* BZIP2 example * BZIP2 example
* UD=307,1,9 * UD=307,0,1,9
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
else if (HDstrcmp(scomp, "UD") == 0) { else if (HDstrcmp(scomp, "UD") == 0) {
l = -1; /* filter number index check */ l = -1; /* filter number index check */
f = -1; /* filter flag index check */
p = -1; /* CD_VAL count check */ p = -1; /* CD_VAL count check */
for (m = 0, q = 0, u = i + 1; u < len; u++, m++, q++) { for (m = 0, q = 0, u = i + 1; u < len; u++, m++, q++) {
if (str[u] == ',') { if (str[u] == ',') {
@ -224,6 +225,10 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
filt->filtn = HDatoi(stype); filt->filtn = HDatoi(stype);
l = 0; l = 0;
} }
else if (f == -1) {
filt->filt_flag = HDstrtoul(stype, NULL, 0);
f = 0;
}
else if (p == -1) { else if (p == -1) {
filt->cd_nelmts = HDstrtoull(stype, NULL, 0); filt->cd_nelmts = HDstrtoull(stype, NULL, 0);
p = 0; p = 0;
@ -240,6 +245,12 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt,
error_msg("filter number parameter is not a digit in <%s>\n", str); error_msg("filter number parameter is not a digit in <%s>\n", str);
HDexit(EXIT_FAILURE); HDexit(EXIT_FAILURE);
} }
else if (!HDisdigit(c) && f == -1) {
if (obj_list)
HDfree(obj_list);
error_msg("filter flag parameter is not a digit in <%s>\n", str);
HDexit(EXIT_FAILURE);
}
stype[q] = c; stype[q] = c;
} /* for u */ } /* for u */
stype[q] = '\0'; stype[q] = '\0';

6
tools/test/h5repack/CMakeTests.cmake
View File

@ -1256,15 +1256,15 @@
############################################################################## ##############################################################################
### P L U G I N T E S T S ### P L U G I N T E S T S
############################################################################## ##############################################################################
ADD_H5_UD_TEST (plugin_version_test 0 h5repack_layout.h5 -v -f UD=260,4,9,${H5_VERS_MAJOR},${H5_VERS_MINOR},${H5_VERS_RELEASE}) ADD_H5_UD_TEST (plugin_version_test 0 h5repack_layout.h5 -v -f UD=260,0,4,9,${H5_VERS_MAJOR},${H5_VERS_MINOR},${H5_VERS_RELEASE})
ADD_H5_UD_TEST (plugin_test 0 h5repack_layout.h5 -v -f UD=257,1,9) ADD_H5_UD_TEST (plugin_test 0 h5repack_layout.h5 -v -f UD=257,0,1,9)
ADD_H5_UD_TEST (plugin_none 0 h5repack_layout.UD.h5 -v -f NONE) ADD_H5_UD_TEST (plugin_none 0 h5repack_layout.UD.h5 -v -f NONE)
# check for no parameters # check for no parameters
set (TESTRETVAL 255) set (TESTRETVAL 255)
if (WIN32) if (WIN32)
set (TESTRETVAL -1) set (TESTRETVAL -1)
endif () endif ()
ADD_H5_CMP_TEST (plugin_zero "" "TEST" ${TESTRETVAL} h5repack_layout.h5 -v -f UD=250,0) ADD_H5_CMP_TEST (plugin_zero "" "TEST" ${TESTRETVAL} h5repack_layout.h5 -v -f UD=250,0,0)
if (HDF5_TEST_VFD) if (HDF5_TEST_VFD)
# Run test with different Virtual File Driver # Run test with different Virtual File Driver

14
tools/test/h5repack/h5repack_plugin.sh.in
View File

@ -1,10 +1,10 @@
#! /bin/sh #! /bin/sh
# #
# Copyright by The HDF Group. # Copyright by The HDF Group.
# All rights reserved. # All rights reserved.
# #
# This file is part of HDF5. The full HDF5 copyright notice, including # This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in # terms governing use, modification, and redistribution, is contained in
# the COPYING file, which can be found at the root of the source code # the COPYING file, which can be found at the root of the source code
# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. # distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
# If you do not have access to either file, you may request a copy from # If you do not have access to either file, you may request a copy from
@ -247,10 +247,10 @@ COPY_TESTFILES_TO_TESTDIR
version_str=`echo @H5_VERSION@ | awk -F"-" '{print $1}' | sed 's/\./,/g'` version_str=`echo @H5_VERSION@ | awk -F"-" '{print $1}' | sed 's/\./,/g'`
# Run the test # Run the test
arg="h5repack_layout.h5 -v -f UD=260,4,9,$version_str" arg="h5repack_layout.h5 -v -f UD=260,0,4,9,$version_str"
TOOLTEST_DUMP plugin_version_test $arg TOOLTEST_DUMP plugin_version_test $arg
arg="h5repack_layout.h5 -v -f UD=257,1,9" arg="h5repack_layout.h5 -v -f UD=257,0,1,9"
TOOLTEST_DUMP plugin_test $arg TOOLTEST_DUMP plugin_test $arg
#arg="h5repack_layout.UD.h5 -v -f NONE" #arg="h5repack_layout.UD.h5 -v -f NONE"

12
tools/test/h5repack/testfiles/h5repack-help.txt
View File

@ -26,7 +26,7 @@ usage: h5repack [OPTIONS] file1 file2
-T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy
-G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size
M - is an integer greater than 1, size of dataset in bytes (default is 0) M - is an integer greater than 1, size of dataset in bytes (default is 0)
E - is a filename. E - is a filename.
S - is an integer S - is an integer
U - is a filename. U - is a filename.
@ -89,8 +89,8 @@ usage: h5repack [OPTIONS] file1 file2
NBIT (no parameter) NBIT (no parameter)
SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type
is either IN or DS is either IN or DS
UD=<filter_number,cd_value_count,value_1[,value_2,...,value_N]> UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
required values for filter_number,cd_value_count,value_1 required values for filter_number,filter_flag,cd_value_count,value_1
optional values for value_2 to value_N optional values for value_2 to value_N
NONE (no parameter) NONE (no parameter)
@ -125,16 +125,16 @@ Examples of use:
Chunked layout, with a layout size of 20x10, to objects dset1 and dset2 Chunked layout, with a layout size of 20x10, to objects dset1 and dset2
and remove filters to objects dset3, dset4, dset5 and remove filters to objects dset3, dset4, dset5
4) h5repack -L -c 10 -s 20:dtype file1 file2 4) h5repack -L -c 10 -s 20:dtype file1 file2
Using latest file format with maximum compact group size of 10 and Using latest file format with maximum compact group size of 10 and
and minimum shared datatype size of 20 and minimum shared datatype size of 20
5) h5repack -f SHUF -f GZIP=1 file1 file2 5) h5repack -f SHUF -f GZIP=1 file1 file2
Add both filters SHUF and GZIP in this order to all datasets Add both filters SHUF and GZIP in this order to all datasets
6) h5repack -f UD=307,1,9 file1 file2 6) h5repack -f UD=307,0,1,9 file1 file2
Add bzip2 filter to all datasets Add bzip2 filter to all datasets