mirror of
https://github.com/HDFGroup/hdf5.git
synced 2024-11-27 02:10:55 +08:00
bc6ab7c0a6
h5repack changes Description: there were some requests to change some minor h5repack features h5repack only made a warning about a non available filter in verbose mode ( -v ) without -v it kept silent, and users sometimes missed this warning the request was that it should print this warning always. so, the new format, is e.g ./h5repack -i test_szip.h5 -o out.h5 Warning: dataset </dset_szip> cannot be read, SZIP filter is not available due to this, and to avoid a lot of these messages in the shell test script, I modified the script h5repack.sh so that it detects the presence of all filters in the environment (previously it only detected SZIP) the test files were also divided in more files , to make the script code easier to follow Solution: Platforms tested: linux AIX (no szip) solaris (no szip, no gzip ) Misc. update:
371 lines
12 KiB
C
371 lines
12 KiB
C
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
* Copyright by the Board of Trustees of the University of Illinois. *
|
|
* All rights reserved. *
|
|
* *
|
|
* This file is part of HDF5. The full HDF5 copyright notice, including *
|
|
* terms governing use, modification, and redistribution, is contained in *
|
|
* the files COPYING and Copyright.html. COPYING can be found at the root *
|
|
* of the source code distribution tree; Copyright.html can be found at the *
|
|
* root level of an installed copy of the electronic HDF5 document set and *
|
|
* is linked from the top-level documents page. It can also be found at *
|
|
* http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have *
|
|
* access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
|
|
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
|
|
|
|
|
|
#ifndef H5REPACK_H__
|
|
#define H5REPACK_H__
|
|
|
|
#include "hdf5.h"
|
|
#include "h5trav.h"
|
|
#include "h5diff.h"
|
|
#include "h5tools.h"
|
|
|
|
#define H5FOPENERROR "unable to open file"
|
|
|
|
#define PFORMAT "%-7s %-7s %-7s\n" /*chunk info, compression info, name*/
|
|
#define PFORMAT1 "%-7s %-7s %-7s" /*chunk info, compression info, name*/
|
|
|
|
#define MAX_NC_NAME 256 /* max length of a name */
|
|
#define MAX_VAR_DIMS 32 /* max per variable dimensions */
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* data structures for command line options
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
/* a list of names */
|
|
typedef struct {
|
|
char obj[MAX_NC_NAME];
|
|
} obj_list_t;
|
|
|
|
/*
|
|
the type of filter and additional parameter
|
|
type can be one of the filters
|
|
H5Z_FILTER_NONE 0, uncompress if compressed
|
|
H5Z_FILTER_DEFLATE 1 , deflation like gzip
|
|
H5Z_FILTER_SHUFFLE 2 , shuffle the data
|
|
H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC
|
|
H5Z_FILTER_SZIP 4 , szip compression
|
|
*/
|
|
|
|
#define CDVALUES 2
|
|
|
|
typedef struct {
|
|
H5Z_filter_t filtn; /* filter identification number */
|
|
int cd_values[CDVALUES]; /* filter client data values */
|
|
} filter_info_t;
|
|
|
|
/* chunk lengths along each dimension and rank */
|
|
typedef struct {
|
|
hsize_t chunk_lengths[MAX_VAR_DIMS];
|
|
int rank;
|
|
} chunk_info_t;
|
|
|
|
/* we currently define a maximum value for the filters array,
|
|
that corresponds to the current library filters */
|
|
#define H5_REPACK_MAX_NFILTERS 4
|
|
|
|
/* information for one object, contains PATH, CHUNK info and FILTER info */
|
|
typedef struct {
|
|
char path[MAX_NC_NAME]; /* name of object */
|
|
filter_info_t filter[H5_REPACK_MAX_NFILTERS]; /* filter array */
|
|
int nfilters; /* current number of filters */
|
|
H5D_layout_t layout; /* layout information */
|
|
chunk_info_t chunk; /* chunk information */
|
|
hid_t refobj_id; /* object ID, references */
|
|
} pack_info_t;
|
|
|
|
/* store a table of all objects */
|
|
typedef struct {
|
|
int size;
|
|
int nelems;
|
|
pack_info_t *objs;
|
|
} pack_opttbl_t;
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* command line options
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
/* all the above, ready to go to the hrepack call */
|
|
typedef struct {
|
|
pack_opttbl_t *op_tbl; /*table with all -c and -f options */
|
|
int all_layout; /*apply the layout to all objects */
|
|
int all_filter; /*apply the filter to all objects */
|
|
filter_info_t filter_g; /*global filter INFO for the ALL case */
|
|
chunk_info_t chunk_g; /*global chunk INFO for the ALL case */
|
|
H5D_layout_t layout_g; /*global layout information for the ALL case */
|
|
int verbose; /*verbose mode */
|
|
hsize_t threshold; /*minimum size to compress, in bytes */
|
|
} pack_opt_t;
|
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* public functions
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
int h5repack (const char* infile, const char* outfile, pack_opt_t *options);
|
|
int h5repack_addfilter (const char* str, pack_opt_t *options);
|
|
int h5repack_addlayout (const char* str, pack_opt_t *options);
|
|
int h5repack_init (pack_opt_t *options, int verbose);
|
|
int h5repack_end (pack_opt_t *options);
|
|
int h5repack_verify (const char *fname,pack_opt_t *options);
|
|
int h5repack_cmpdcpl (const char *fname1,
|
|
const char *fname2);
|
|
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* private functions
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
|
|
int check_objects(const char* fname,
|
|
pack_opt_t *options);
|
|
|
|
int copy_objects(const char* fnamein,
|
|
const char* fnameout,
|
|
pack_opt_t *options);
|
|
|
|
void print_objlist(const char *filename,
|
|
int nobjects,
|
|
trav_info_t *travi );
|
|
|
|
int do_copy_objects(hid_t fidin,
|
|
hid_t fidout,
|
|
trav_table_t *travt,
|
|
pack_opt_t *options);
|
|
|
|
int copy_attr(hid_t loc_in,
|
|
hid_t loc_out,
|
|
pack_opt_t *options
|
|
);
|
|
|
|
int do_copy_refobjs(hid_t fidin,
|
|
hid_t fidout,
|
|
trav_table_t *travt,
|
|
pack_opt_t *options); /* repack options */
|
|
|
|
|
|
|
|
void read_info(const char *filename,pack_opt_t *options);
|
|
void init_packobject(pack_info_t *obj);
|
|
int print_filters(hid_t dcpl_id);
|
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* filters
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
int filter_this(const char* name,
|
|
pack_opt_t *options,
|
|
pack_info_t *pack); /* info about object to filter */
|
|
|
|
int apply_filters(const char* name, /* object name from traverse list */
|
|
int rank, /* rank of dataset */
|
|
hsize_t *dims, /* dimensions of dataset */
|
|
hid_t dcpl_id, /* dataset creation property list */
|
|
hid_t type_id, /* datatype */
|
|
pack_opt_t *options, /* repack options */
|
|
pack_info_t *obj); /* info about object to filter */
|
|
|
|
int has_filter(hid_t dcpl_id,
|
|
H5Z_filter_t filtnin);
|
|
|
|
int check_szip_params( unsigned bits_per_pixel,
|
|
unsigned pixels_per_block,
|
|
unsigned pixels_per_scanline,
|
|
hsize_t image_pixels);
|
|
|
|
int check_szip(hid_t type_id, /* dataset datatype */
|
|
int rank, /* chunk rank */
|
|
hsize_t *dims, /* chunk dims */
|
|
unsigned szip_options_mask /*IN*/,
|
|
unsigned *szip_pixels_per_block /*IN,OUT*/,
|
|
pack_opt_t *options);
|
|
|
|
int can_read(const char* name, /* object name from traverse list */
|
|
hid_t dcpl_id, /* dataset creation property list */
|
|
pack_opt_t *options); /* repack options */
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* layout functions
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
int has_layout(hid_t dcpl_id,
|
|
pack_info_t *obj);
|
|
|
|
int layout_this(hid_t dcpl_id, /* DCPL from input object */
|
|
const char* name, /* object name from traverse list */
|
|
pack_opt_t *options, /* repack options */
|
|
pack_info_t *pack /*OUT*/) /* object to apply layout */;
|
|
|
|
int apply_layout(hid_t dcpl_id,
|
|
pack_info_t *pack); /* info about object */
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* options table
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
int options_table_init( pack_opttbl_t **tbl );
|
|
int options_table_free( pack_opttbl_t *table );
|
|
int options_add_layout( obj_list_t *obj_list,
|
|
int n_objs,
|
|
pack_info_t *pack,
|
|
pack_opttbl_t *table );
|
|
int options_add_filter ( obj_list_t *obj_list,
|
|
int n_objs,
|
|
filter_info_t filt,
|
|
pack_opttbl_t *table );
|
|
pack_info_t* options_get_object( const char *path,
|
|
pack_opttbl_t *table);
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* parse functions
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
obj_list_t* parse_filter(const char *str,
|
|
int *n_objs,
|
|
filter_info_t *filt,
|
|
pack_opt_t *options);
|
|
|
|
obj_list_t* parse_layout(const char *str,
|
|
int *n_objs,
|
|
pack_info_t *pack, /* info about object */
|
|
pack_opt_t *options);
|
|
|
|
const char* get_sfilter (H5Z_filter_t filtn);
|
|
int parse_number(char *str);
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* tests
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
#define FNAME1 "test1.h5"
|
|
#define FNAME1OUT "test1.out.h5"
|
|
#define FNAME2 "test2.h5"
|
|
#define FNAME2OUT "test2.out.h5"
|
|
#define FNAME3 "test3.h5"
|
|
#define FNAME3OUT "test3.out.h5"
|
|
#define FNAME4 "test4.h5"
|
|
#define FNAME4OUT "test4.out.h5"
|
|
#define FNAME5 "test5.h5"
|
|
#define FNAME5OUT "test5.out.h5"
|
|
#define FNAME6 "test6.h5"
|
|
#define FNAME7 "test_szip.h5"
|
|
#define FNAME8 "test_deflate.h5"
|
|
#define FNAME9 "test_shuffle.h5"
|
|
#define FNAME10 "test_fletcher32.h5"
|
|
#define FNAME11 "test_all.h5"
|
|
#define FNAME7OUT "test_szip.out.h5"
|
|
#define FNAME8OUT "test_deflate.out.h5"
|
|
#define FNAME9OUT "test_shuffle.out.h5"
|
|
#define FNAME10OUT "test_fletcher32.out.h5"
|
|
#define FNAME11OUT "test_all.out.h5"
|
|
|
|
int make_testfiles(void);
|
|
|
|
int write_dset( hid_t loc_id,
|
|
int rank,
|
|
hsize_t *dims,
|
|
const char *dset_name,
|
|
hid_t type_id,
|
|
void *buf );
|
|
int write_attr(hid_t loc_id,
|
|
int rank,
|
|
hsize_t *dims,
|
|
const char *attr_name,
|
|
hid_t type_id,
|
|
void *buf);
|
|
void write_attr_in(hid_t loc_id,
|
|
const char* dset_name, /* for saving reference to dataset*/
|
|
hid_t fid, /* for reference create */
|
|
int make_diffs /* flag to modify data buffers */);
|
|
void write_null_attr(hid_t loc_id);
|
|
void write_dset_in(hid_t loc_id,
|
|
const char* dset_name, /* for saving reference to dataset*/
|
|
hid_t file_id,
|
|
int make_diffs /* flag to modify data buffers */);
|
|
void write_null_dset(hid_t loc_id);
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* tests utils
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
int make_dset(hid_t loc_id,
|
|
const char *name,
|
|
hid_t sid,
|
|
hid_t dcpl,
|
|
void *buf);
|
|
|
|
int make_attr(hid_t loc_id,
|
|
int rank,
|
|
hsize_t *dims,
|
|
const char *attr_name,
|
|
hid_t type_id,
|
|
void *buf);
|
|
|
|
|
|
|
|
/*-------------------------------------------------------------------------
|
|
* check SZIP parameters
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
|
|
typedef struct
|
|
{
|
|
int compression_mode;
|
|
int options_mask;
|
|
unsigned bits_per_pixel;
|
|
unsigned pixels_per_block;
|
|
unsigned pixels_per_scanline;
|
|
hsize_t pixels;
|
|
|
|
}szip_comp_t;
|
|
|
|
/* for SZIP */
|
|
#if !defined (NN_OPTION_MASK)
|
|
#define NN_OPTION_MASK 32
|
|
#endif
|
|
#if !defined (RAW_OPTION_MASK)
|
|
#define RAW_OPTION_MASK 128
|
|
#endif
|
|
#if !defined (MAX_BLOCKS_PER_SCANLINE)
|
|
#define MAX_BLOCKS_PER_SCANLINE 128
|
|
#endif
|
|
#if !defined (MAX_PIXELS_PER_BLOCK)
|
|
#define MAX_PIXELS_PER_BLOCK 32
|
|
#endif
|
|
#if !defined (MAX_PIXELS_PER_SCANLINE)
|
|
#define MAX_PIXELS_PER_SCANLINE (MAX_BLOCKS_PER_SCANLINE)*(MAX_PIXELS_PER_BLOCK)
|
|
#endif
|
|
#if !defined (NN_MODE)
|
|
#define NN_MODE 1
|
|
#endif
|
|
|
|
|
|
|
|
#endif /* H5REPACK_H__ */
|