mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-30 15:32:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

HDFFV-8611 change h5repack to save the root group creation order

Browse Source 
Added test and new arguments to control the the input file parsing.
This commit is contained in:
Allen Byrne 2017-05-22 13:10:40 -05:00
parent 8861b5a494
commit cdcef539a3
13 changed files with 191 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST
View File

@ -2432,6 +2432,7 @@
./tools/test/h5repack/testfiles/h5repack_soffset.h5
./tools/test/h5repack/testfiles/h5repack_szip.h5
./tools/test/h5repack/testfiles/ublock.bin
./tools/test/h5repack/testfiles/crtorder.tordergr.h5.ddl
./tools/test/h5repack/testfiles/deflate_limit.h5repack_layout.h5.ddl
./tools/test/h5repack/testfiles/plugin_none.h5repack_layout.UD.h5.tst
./tools/test/h5repack/testfiles/plugin_test.h5repack_layout.h5.tst

4
tools/lib/h5tools.c
View File

@ -51,6 +51,10 @@ unsigned packed_data_length; /* length of packed bits to display */
unsigned long long packed_data_mask; /* mask in which packed bits to display */
int enable_error_stack= FALSE; /* re-enable error stack */
/* sort parameters */
H5_index_t sort_by = H5_INDEX_NAME; /*sort_by [creation_order | name] */
H5_iter_order_t sort_order = H5_ITER_INC; /*sort_order [ascending | descending] */
/* module-scoped variables */
static int h5tools_init_g; /* if h5tools lib has been initialized */

4
tools/lib/h5tools.h
View File

@ -568,6 +568,10 @@ H5TOOLS_DLLVAR int oid_output; /* oid output */
H5TOOLS_DLLVAR int data_output; /* data output */
H5TOOLS_DLLVAR int attr_data_output; /* attribute data output */
/* sort parameters */
H5TOOLS_DLLVAR H5_index_t sort_by; /*sort_by [creation_order | name] */
H5TOOLS_DLLVAR H5_iter_order_t sort_order; /*sort_order [ascending | descending] */
/* things to display or which are set via command line parameters */
H5TOOLS_DLLVAR int enable_error_stack; /* re-enable error stack */

4
tools/src/h5dump/h5dump.h
View File

@ -78,10 +78,6 @@ int include_attrs = TRUE; /* Display attributes */
int display_vds_first = FALSE; /* vds display to all by default*/
int vds_gap_size = 0; /* vds skip missing files default is none */
/* sort parameters */
H5_index_t sort_by = H5_INDEX_NAME; /*sort_by [creation_order | name] */
H5_iter_order_t sort_order = H5_ITER_INC; /*sort_order [ascending | descending] */
#define PACKED_BITS_MAX 8 /* Maximum number of packed-bits to display */
#define PACKED_BITS_SIZE_MAX (8*sizeof(long long)) /* Maximum bits size of integer types of packed-bits */
/* mask list for packed bits */

4
tools/src/h5dump/h5dump_extern.h
View File

@ -77,10 +77,6 @@ extern int include_attrs; /* Display attributes */
extern int display_vds_first; /* vds display to first missing */
extern int vds_gap_size; /* vds skip missing files */
/* sort parameters */
extern H5_index_t sort_by; /*sort_by [creation_order | name] */
extern H5_iter_order_t sort_order; /*sort_order [ascending | descending] */
#define PACKED_BITS_MAX 8 /* Maximum number of packed-bits to display */
#define PACKED_BITS_SIZE_MAX 8*sizeof(long long) /* Maximum bits size of integer types of packed-bits */
/* mask list for packed bits */

2
tools/src/h5repack/h5repack.c
View File

@ -784,6 +784,8 @@ static int check_objects(const char* fname, pack_opt_t *options) {
*-------------------------------------------------------------------------
*/
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
/* init table */
trav_table_init(&travt);

19
tools/src/h5repack/h5repack_copy.c
View File

@ -103,6 +103,7 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
hbool_t in_persist; /* Persist free-space status from input file */
hsize_t in_threshold; /* Free-space section threshold from input file */
hsize_t in_pagesize; /* File space page size from input file */
unsigned crt_order_flags; /* group creation order flag */
/*-------------------------------------------------------------------------
* open input file
@ -116,6 +117,8 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
/* get user block size and file space strategy/persist/threshold */
{
hid_t fcpl_in; /* file creation property list ID for input file */
hid_t grp_in = -1; /* group ID */
hid_t gcpl_in = -1; /* group creation property list */
if ((fcpl_in = H5Fget_create_plist(fidin)) < 0) {
error_msg("failed to retrieve file creation property list\n");
@ -138,6 +141,17 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
error_msg("failed to retrieve file space threshold\n");
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
}
/* open root group */
if ((grp_in = H5Gopen2(fidin, "/", H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gopen2 failed");
/* get root group creation property list */
if ((gcpl_in = H5Gget_create_plist(grp_in)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_create_plist failed");
/* query and set the group creation properties */
if (H5Pget_link_creation_order(gcpl_in, &crt_order_flags) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_link_creation_order failed");
if (H5Pclose(fcpl_in) < 0) {
error_msg("failed to close property list\n");
@ -309,6 +323,9 @@ print_user_block(fnamein, fidin);
}
}
if(H5Pset_link_creation_order(fcpl, crt_order_flags ) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_creation_order failed");
/* Set file space info to those from input file */
set_strategy = in_strategy;
set_persist = in_persist;
@ -375,6 +392,8 @@ print_user_block(fnamein, fidin);
*-------------------------------------------------------------------------
*/
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
/* init table */
trav_table_init(&travt);

79
tools/src/h5repack/h5repack_main.c
View File

@ -31,7 +31,7 @@ const char *outfile = NULL;
* Command-line options: The user can specify short or long-named
* parameters.
*/
static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:P:T:G:E";
static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:P:T:G:q:z:E";
static struct long_options l_opts[] = {
{ "help", no_arg, 'h' },
{ "version", no_arg, 'V' },
@ -56,6 +56,8 @@ static struct long_options l_opts[] = {
{ "fs_persist", require_arg, 'P' },
{ "fs_threshold", require_arg, 'T' },
{ "fs_pagesize", require_arg, 'G' },
{ "sort_by", require_arg, 'q' },
{ "sort_order", require_arg, 'z' },
{ "enable-error-stack", no_arg, 'E' },
{ NULL, 0, '\0' }
};
@ -90,6 +92,8 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " -M A, --metadata_block_size=A Metadata block size for H5Pset_meta_block_size\n");
PRINTVALSTREAM(rawoutstream, " -t T, --threshold=T Threshold value for H5Pset_alignment\n");
PRINTVALSTREAM(rawoutstream, " -a A, --alignment=A Alignment value for H5Pset_alignment\n");
PRINTVALSTREAM(rawoutstream, " -q Q, --sort_by=Q Sort groups and attributes by index Q\n");
PRINTVALSTREAM(rawoutstream, " -z Z, --sort_order=Z Sort groups and attributes by order Z\n");
PRINTVALSTREAM(rawoutstream, " -f FILT, --filter=FILT Filter type\n");
PRINTVALSTREAM(rawoutstream, " -l LAYT, --layout=LAYT Layout type\n");
PRINTVALSTREAM(rawoutstream, " -S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy\n");
@ -103,6 +107,8 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " U - is a filename.\n");
PRINTVALSTREAM(rawoutstream, " T - is an integer\n");
PRINTVALSTREAM(rawoutstream, " A - is an integer greater than zero\n");
PRINTVALSTREAM(rawoutstream, " Q - is the sort index type for the input file. It can be \"name\" or \"creation_order\" (default)\n");
PRINTVALSTREAM(rawoutstream, " Z - is the sort order type for the input file. It can be \"descending\" or \"ascending\" (default)\n");
PRINTVALSTREAM(rawoutstream, " B - is the user block size, any value that is 512 or greater and is\n");
PRINTVALSTREAM(rawoutstream, " a power of 2 (1024 default)\n");
PRINTVALSTREAM(rawoutstream, " F - is the shared object header message type, any of <dspace|dtype|fill|\n");
@ -369,6 +375,52 @@ done:
return ret_value;
}
/*-------------------------------------------------------------------------
* Function: set_sort_by
*
* Purpose: set the "by" form of sorting by translating from a string input
* parameter to a H5_index_t return value
* current sort values are [creation_order | name]
*
* Return: H5_index_t form of sort or H5_INDEX_UNKNOWN if none found
*-------------------------------------------------------------------------
*/
static H5_index_t
set_sort_by(const char *form)
{
H5_index_t idx_type = H5_INDEX_UNKNOWN;
if (HDstrcmp(form,"name")==0) /* H5_INDEX_NAME */
idx_type = H5_INDEX_NAME;
else if (HDstrcmp(form,"creation_order")==0) /* H5_INDEX_CRT_ORDER */
idx_type = H5_INDEX_CRT_ORDER;
return idx_type;
}
/*-------------------------------------------------------------------------
* Function: set_sort_order
*
* Purpose: set the order of sorting by translating from a string input
* parameter to a H5_iter_order_t return value
* current order values are [ascending | descending ]
*
* Return: H5_iter_order_t form of order or H5_ITER_UNKNOWN if none found
*-------------------------------------------------------------------------
*/
static H5_iter_order_t
set_sort_order(const char *form)
{
H5_iter_order_t iter_order = H5_ITER_UNKNOWN;
if (HDstrcmp(form,"ascending")==0) /* H5_ITER_INC */
iter_order = H5_ITER_INC;
else if (HDstrcmp(form,"descending")==0) /* H5_ITER_DEC */
iter_order = H5_ITER_DEC;
return iter_order;
}
/*-------------------------------------------------------------------------
* Function: parse_command_line
*
@ -553,14 +605,14 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
break;
case 'P':
options->fs_persist = HDatoi( opt_arg );
options->fs_persist = HDatoi(opt_arg);
if(options->fs_persist == 0)
/* To distinguish the "specified" zero value */
options->fs_persist = -1;
break;
case 'T':
options->fs_threshold = HDatol( opt_arg );
options->fs_threshold = HDatol(opt_arg);
if(options->fs_threshold == 0)
/* To distinguish the "specified" zero value */
options->fs_threshold = -1;
@ -573,6 +625,24 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
options->fs_pagesize = -1;
break;
case 'q':
if((sort_by = set_sort_by(opt_arg)) < 0) {
error_msg(" failed to set sort by form <%s>\n", opt_arg);
h5tools_setstatus(EXIT_FAILURE);
ret_value = -1;
goto done;
}
break;
case 'z':
if((sort_order = set_sort_order(opt_arg)) < 0) {
error_msg(" failed to set sort order form <%s>\n", opt_arg);
h5tools_setstatus(EXIT_FAILURE);
ret_value = -1;
goto done;
}
break;
case 'E':
enable_error_stack = TRUE;
break;
@ -644,6 +714,9 @@ int main(int argc, const char **argv)
/* initialize options */
h5repack_init(&options, 0, FALSE);
/* Initialize default indexing options */
sort_by = H5_INDEX_CRT_ORDER;
if (parse_command_line(argc, argv, &options) < 0)
goto done;

5
tools/src/h5repack/h5repack_verify.c
View File

@ -122,6 +122,8 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options
if(options->all_filter == 1 || options->all_layout == 1)
{
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
/* init table */
trav_table_init(&travt);
@ -455,6 +457,9 @@ int h5repack_cmp_pl(const char *fname1,
* get file table list of objects
*-------------------------------------------------------------------------
*/
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
/* init table */
trav_table_init(&trav);
if(h5trav_gettable(fid1, trav) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "h5trav_gettable failed");

9
tools/test/h5repack/CMakeTests.cmake
View File

@ -90,6 +90,7 @@
${HDF5_TOOLS_DIR}/testfiles/tfamily00008.h5
${HDF5_TOOLS_DIR}/testfiles/tfamily00009.h5
${HDF5_TOOLS_DIR}/testfiles/tfamily00010.h5
${HDF5_TOOLS_DIR}/testfiles/tordergr.h5
# tools/testfiles/vds
${HDF5_TOOLS_DIR}/testfiles/vds/1_a.h5
${HDF5_TOOLS_DIR}/testfiles/vds/1_b.h5
@ -137,6 +138,7 @@
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_compa-v.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst
${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl
)
foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES})
@ -312,7 +314,7 @@
NAME H5REPACK_DMP-h5dump-${testname}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5dump>"
-D "TEST_ARGS:STRING=-pH;out-${testname}.${resultfile}"
-D "TEST_ARGS:STRING=-q;creation_order;-pH;out-${testname}.${resultfile}"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
-D "TEST_OUTPUT=${resultfile}-${testname}.out"
-D "TEST_EXPECT=${resultcode}"
@ -965,6 +967,11 @@
endif ()
ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg})
#crtorder
set (arg tordergr.h5 -L)
set (TESTTYPE "TEST")
ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg})
#########################################################
# layout options (these files have no filters)
#########################################################

36
tools/test/h5repack/testfiles/crtorder.tordergr.h5.ddl Normal file
View File

@ -0,0 +1,36 @@
HDF5 "out-crtorder.tordergr.h5" {
GROUP "/" {
GROUP "2" {
GROUP "a" {
GROUP "a1" {
}
GROUP "a2" {
GROUP "a21" {
}
GROUP "a22" {
}
}
}
GROUP "b" {
}
GROUP "c" {
}
}
GROUP "1" {
GROUP "c" {
}
GROUP "b" {
}
GROUP "a" {
GROUP "a1" {
}
GROUP "a2" {
GROUP "a22" {
}
GROUP "a21" {
}
}
}
}
}
}

4
tools/test/h5repack/testfiles/h5repack-help.txt
View File

@ -17,6 +17,8 @@ usage: h5repack [OPTIONS] file1 file2
-M A, --metadata_block_size=A Metadata block size for H5Pset_meta_block_size
-t T, --threshold=T Threshold value for H5Pset_alignment
-a A, --alignment=A Alignment value for H5Pset_alignment
-q Q, --sort_by=Q Sort groups and attributes by index Q
-z Z, --sort_order=Z Sort groups and attributes by order Z
-f FILT, --filter=FILT Filter type
-l LAYT, --layout=LAYT Layout type
-S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy
@ -30,6 +32,8 @@ usage: h5repack [OPTIONS] file1 file2
U - is a filename.
T - is an integer
A - is an integer greater than zero
Q - is the sort index type for the input file. It can be "name" or "creation_order" (default)
Z - is the sort order type for the input file. It can be "descending" or "ascending" (default)
B - is the user block size, any value that is 512 or greater and is
a power of 2 (1024 default)
F - is the shared object header message type, any of <dspace|dtype|fill|