mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-18 15:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r14320] Added a function that matches the filters on the dataset's DCPL and in the requested list

Browse Source 
Used to verify if the filters requested are present in the output dataset

tested: windows, linux, solaris
This commit is contained in:
Pedro Vicente Nunes 2007-12-04 14:59:54 -05:00
parent 529d974dff
commit a5ba6ac397
5 changed files with 291 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/h5repack/h5repack.c
View File

@ -101,6 +101,7 @@ int h5repack_init (pack_opt_t *options,
for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++)
{
options->filter_g[n].filtn = -1;
options->filter_g[n].cd_nelmts = -1;
for ( k = 0; k < CDVALUES; k++)
options->filter_g[n].cd_values[k] = -1;
}

20
tools/h5repack/h5repack.h
View File

@ -51,11 +51,12 @@ typedef struct {
H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*/
#define CDVALUES 2
#define CDVALUES 4 /* SZIP returns 4 values */
typedef struct {
H5Z_filter_t filtn; /* filter identification number */
int cd_values[CDVALUES]; /* filter client data values */
size_t cd_nelmts; /* filter client number of values */
} filter_info_t;
/* chunk lengths along each dimension and rank */
@ -96,23 +97,10 @@ typedef struct {
pack_opttbl_t *op_tbl; /*table with all -c and -f options */
int all_layout; /*apply the layout to all objects */
int all_filter; /*apply the filter to all objects */
#if defined (NOT_ALL)
filter_info_t filter_g; /*global filter INFO for the ALL case */
#else
filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */
int n_filter_g; /*number of global filters */
#endif
filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */
int n_filter_g; /*number of global filters */
chunk_info_t chunk_g; /*global chunk INFO for the ALL case */
H5D_layout_t layout_g; /*global layout information for the ALL case */
int verbose; /*verbose mode */
hsize_t threshold; /*minimum size to compress, in bytes */
int use_native; /*use a native type in write */

16
tools/h5repack/h5repack_filters.c
View File

@ -265,12 +265,12 @@ int apply_filters(const char* name, /* object name from traverse list */
/*-------------------------------------------------------------------------
* the type of filter and additional parameter
* type can be one of the filters
* H5Z_FILTER_NONE 0, uncompress if compressed
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
* H5Z_FILTER_NONE 0 , uncompress if compressed
* H5Z_FILTER_DEFLATE 1 , deflation like gzip
* H5Z_FILTER_SHUFFLE 2 , shuffle the data
* H5Z_FILTER_FLETCHER32 3 , fletcher32 checksum of EDC
* H5Z_FILTER_SZIP 4 , szip compression
* H5Z_FILTER_NBIT 5 , nbit compression
* H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression
*-------------------------------------------------------------------------
*/
@ -278,10 +278,10 @@ int apply_filters(const char* name, /* object name from traverse list */
if (obj.nfilters)
{
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* filters require CHUNK layout; if we do not have one define a default
*-------------------------------------------------------------------------
*/
*/
if (obj.layout==-1)
{
obj.chunk.rank=rank;

92
tools/h5repack/h5repack_parse.c
View File

@ -74,7 +74,7 @@ obj_list_t* parse_filter(const char *str,
*is_glb = 0;
/* check for the end of object list and number of objects */
for ( i=0, n=0; i<len; i++)
for ( i = 0, n = 0; i < len; i++)
{
c = str[i];
if ( c==':' )
@ -87,14 +87,15 @@ obj_list_t* parse_filter(const char *str,
}
}
if (end_obj==-1) { /* missing : */
if (end_obj==-1) /* missing : */
{
/* apply to all objects */
options->all_filter=1;
*is_glb = 1;
}
n++;
obj_list=malloc(n*sizeof(obj_list_t));
obj_list = malloc(n*sizeof(obj_list_t));
if (obj_list==NULL)
{
error_msg(progname, "could not allocate object list\n");
@ -103,10 +104,10 @@ obj_list_t* parse_filter(const char *str,
*n_objs=n;
/* get object list */
for ( j=0, k=0, n=0; j<end_obj; j++,k++)
for ( j = 0, k = 0, n = 0; j < end_obj; j++, k++)
{
c = str[j];
sobj[k]=c;
sobj[k] = c;
if ( c==',' || j==end_obj-1)
{
if ( c==',') sobj[k]='\0'; else sobj[k+1]='\0';
@ -182,6 +183,7 @@ obj_list_t* parse_filter(const char *str,
exit(1);
}
}
}
@ -280,36 +282,48 @@ obj_list_t* parse_filter(const char *str,
scomp[k+1]='\0';
no_param=1;
}
/*-------------------------------------------------------------------------
* translate from string to filter symbol
*-------------------------------------------------------------------------
*/
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_NONE
*-------------------------------------------------------------------------
*/
if (strcmp(scomp,"NONE")==0)
{
filt->filtn=H5Z_FILTER_NONE;
filt->cd_nelmts = 0;
}
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE
*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE
*-------------------------------------------------------------------------
*/
else if (strcmp(scomp,"GZIP")==0)
{
filt->filtn=H5Z_FILTER_DEFLATE;
if (no_param) { /*no more parameters, GZIP must have parameter */
filt->cd_nelmts = 1;
if (no_param)
{ /*no more parameters, GZIP must have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "missing compression parameter in <%s>\n",str);
exit(1);
}
}
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP
*-------------------------------------------------------------------------
*/
else if (strcmp(scomp,"SZIP")==0)
{
filt->filtn=H5Z_FILTER_SZIP;
if (no_param) { /*no more parameters, SZIP must have parameter */
filt->cd_nelmts = 2;
if (no_param)
{ /*no more parameters, SZIP must have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "missing compression parameter in <%s>\n",str);
exit(1);
@ -323,46 +337,54 @@ obj_list_t* parse_filter(const char *str,
else if (strcmp(scomp,"SHUF")==0)
{
filt->filtn=H5Z_FILTER_SHUFFLE;
if (m>0){ /*shuffle does not have parameter */
filt->cd_nelmts = 0;
if (m>0)
{ /*shuffle does not have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "extra parameter in SHUF <%s>\n",str);
exit(1);
}
}
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_FLETCHER32
*-------------------------------------------------------------------------
*/
else if (strcmp(scomp,"FLET")==0)
{
filt->filtn=H5Z_FILTER_FLETCHER32;
if (m>0){ /*shuffle does not have parameter */
filt->cd_nelmts = 0;
if (m>0)
{ /*shuffle does not have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "extra parameter in FLET <%s>\n",str);
exit(1);
}
}
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_NBIT
*-------------------------------------------------------------------------
*/
else if (strcmp(scomp,"NBIT")==0)
{
filt->filtn=H5Z_FILTER_NBIT;
if (m>0){ /*nbit does not have parameter */
filt->cd_nelmts = 0;
if (m>0)
{ /*nbit does not have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "extra parameter in NBIT <%s>\n",str);
exit(1);
}
}
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET
*-------------------------------------------------------------------------
*/
else if (strcmp(scomp,"SOFF")==0)
{
filt->filtn=H5Z_FILTER_SCALEOFFSET;
if (no_param) { /*no more parameters, SOFF must have parameter */
filt->cd_nelmts = 2;
if (no_param)
{ /*no more parameters, SOFF must have parameter */
if (obj_list) free(obj_list);
error_msg(progname, "missing compression parameter in <%s>\n",str);
exit(1);
@ -376,43 +398,63 @@ obj_list_t* parse_filter(const char *str,
}
} /*i*/
/*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* check valid parameters
*-------------------------------------------------------------------------
*/
switch (filt->filtn)
{
/*-------------------------------------------------------------------------
* H5Z_FILTER_DEFLATE
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_DEFLATE:
if (filt->cd_values[0]<0 || filt->cd_values[0]>9 ){
if (filt->cd_values[0]<0 || filt->cd_values[0]>9 )
{
if (obj_list) free(obj_list);
error_msg(progname, "invalid compression parameter in <%s>\n",str);
exit(1);
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SZIP
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SZIP:
pixels_per_block=filt->cd_values[0];
if ((pixels_per_block%2)==1) {
if ((pixels_per_block%2)==1)
{
if (obj_list) free(obj_list);
error_msg(progname, "pixels_per_block is not even in <%s>\n",str);
exit(1);
}
if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK) {
if (pixels_per_block>H5_SZIP_MAX_PIXELS_PER_BLOCK)
{
if (obj_list) free(obj_list);
error_msg(progname, "pixels_per_block is too large in <%s>\n",str);
exit(1);
}
if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) ) {
if ( (strcmp(smask,"NN")!=0) && (strcmp(smask,"EC")!=0) )
{
if (obj_list) free(obj_list);
error_msg(progname, "szip mask must be 'NN' or 'EC' \n");
exit(1);
}
break;
/*-------------------------------------------------------------------------
* H5Z_FILTER_SCALEOFFSET
*-------------------------------------------------------------------------
*/
case H5Z_FILTER_SCALEOFFSET:
if (filt->cd_values[0]<0 ){
if (filt->cd_values[0]<0 )
{
if (obj_list) free(obj_list);
error_msg(progname, "invalid compression parameter in <%s>\n",str);
exit(1);

232
tools/h5repack/h5repack_verify.c
View File

@ -20,6 +20,8 @@
extern char *progname;
static int has_filter(hid_t dcpl_id, H5Z_filter_t filtnin);
static int has_layout(hid_t dcpl_id,pack_info_t *obj);
static int has_filters(hid_t dcpl_id, pack_opt_t *options);
static int filtcmp( filter_info_t f1, filter_info_t f2);
@ -46,9 +48,9 @@ int h5repack_verify(const char *fname,
hid_t dcpl_id = -1; /* dataset creation property list ID */
hid_t space_id = -1; /* space ID */
unsigned int i;
int j;
trav_table_t *travt = NULL;
int have = 1;
int j;
trav_table_t *travt = NULL;
int ok = 1;
/* open the file */
if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 )
@ -77,7 +79,7 @@ int h5repack_verify(const char *fname,
for(j = 0; j < obj->nfilters; j++)
{
if(has_filter(dcpl_id, obj->filter[j].filtn) == 0)
have = 0;
ok = 0;
}
@ -87,11 +89,11 @@ int h5repack_verify(const char *fname,
*-------------------------------------------------------------------------
*/
if((obj->layout != -1) && (has_layout(dcpl_id, obj) == 0))
have = 0;
ok = 0;
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if(H5Pclose(dcpl_id) < 0)
goto error;
@ -142,13 +144,8 @@ int h5repack_verify(const char *fname,
*/
if(options->all_filter == 1)
{
int k;
for (k = 0; k < options->n_filter_g; k++ )
{
if (has_filter(dcpl_id, options->filter_g[k].filtn) == 0)
have = 0;
}
if (has_filters(dcpl_id, options) == 0)
ok = 0;
}
/*-------------------------------------------------------------------------
@ -162,7 +159,7 @@ int h5repack_verify(const char *fname,
pack.layout = options->layout_g;
pack.chunk = options->chunk_g;
if(has_layout(dcpl_id, &pack) == 0)
have = 0;
ok = 0;
}
@ -192,7 +189,7 @@ int h5repack_verify(const char *fname,
if (H5Fclose(fid) < 0)
return -1;
return have;
return ok;
error:
H5E_BEGIN_TRY {
@ -320,10 +317,6 @@ int has_layout(hid_t dcpl_id,
return 1;
}
/*-------------------------------------------------------------------------
* Function: h5repack_cmpdcpl
*
@ -471,3 +464,200 @@ error:
return -1;
}
/*-------------------------------------------------------------------------
* Function: has_filters
*
* Purpose: verify if all requested filters are present in the
* property list DCPL_ID
*
* Return: 1 has, 0 does not, -1 error
*
* Programmer: Pedro Vicente, pvn@hdfgroup.org
*
* Date: December 3, 2007
*
*-------------------------------------------------------------------------
*/
static int has_filters(hid_t dcpl_id, pack_opt_t *options)
{
unsigned nfilters_dcpl; /* number of filters in DCPL*/
unsigned nfilters_opt; /* number of filters in OPTIONS*/
unsigned filt_flags; /* filter flags */
H5Z_filter_t filtn; /* filter identification number */
unsigned cd_values[20]; /* filter client data values */
size_t cd_nelmts; /* filter client number of values */
char f_name[256]; /* filter name */
int have = 0; /* flag, filter is present */
unsigned i, j; /* index */
filter_info_t filter_dcpl[H5_REPACK_MAX_NFILTERS]; /* filter array in the DCPL*/
filter_info_t filter_opt[H5_REPACK_MAX_NFILTERS]; /* filter array in options */
/* get information about filters */
if((nfilters_dcpl = H5Pget_nfilters(dcpl_id)) < 0)
return -1;
/* if we do not have filters and the requested filter is NONE, return 1 */
if(!nfilters_dcpl &&
options->n_filter_g==1 &&
options->filter_g[0].filtn == H5Z_FILTER_NONE )
return 1;
/*-------------------------------------------------------------------------
* build a list with DCPL filters
*-------------------------------------------------------------------------
*/
for( i = 0; i < nfilters_dcpl; i++)
{
cd_nelmts = NELMTS(cd_values);
filtn = H5Pget_filter2(dcpl_id, i, &filt_flags, &cd_nelmts,
cd_values, sizeof(f_name), f_name, NULL);
filter_dcpl[i].filtn = filtn;
filter_dcpl[i].cd_nelmts = cd_nelmts;
for( j = 0; j < cd_nelmts; j++)
{
filter_dcpl[i].cd_values[j] = cd_values[j];
}
}
/*-------------------------------------------------------------------------
* build a list with options filters
*-------------------------------------------------------------------------
*/
nfilters_opt = options->n_filter_g;
for( i = 0; i < nfilters_opt; i++)
{
filter_opt[i].filtn = options->filter_g[i].filtn;
filter_opt[i].cd_nelmts = options->filter_g[i].cd_nelmts;
for( j = 0; j < options->filter_g[i].cd_nelmts; j++)
{
filter_opt[i].cd_values[j] = options->filter_g[i].cd_values[j];
}
}
/*-------------------------------------------------------------------------
* match the 2 lists
*-------------------------------------------------------------------------
*/
if (nfilters_dcpl != nfilters_opt)
return 0;
for( i = 0; i < nfilters_opt; i++)
{
/* criteria is filter compare, returns same as strcmp */
if ( filtcmp( filter_dcpl[i], filter_opt[i] ) != 0 )
return 0;
}
return 1;
}
/*-------------------------------------------------------------------------
* Function: filtcmp
*
* Purpose: compare 2 filters
*
* Return: same as strcmp:
* < 0 F1 "less" than F2
* 0 F1 identical to F2
* > 0 F1 "greater" than F2
*
* Programmer: Pedro Vicente, pvn@hdfgroup.org
*
* Date: December 3, 2007
*
*-------------------------------------------------------------------------
*/
static int filtcmp( filter_info_t f1 /*DCPL*/, filter_info_t f2 /*OPT*/)
{
unsigned i;
/*-------------------------------------------------------------------------
* compare cd_nelmts and cd_values
*-------------------------------------------------------------------------
*/
if ( f1.filtn == 4 ) /* SZIP special case */
{
if ( f1.cd_nelmts != 4 && f2.cd_nelmts != 2 )
return -1;
if ( f2.cd_values[0] != f1.cd_values[2] &&
f2.cd_values[1] != f1.cd_values[1] )
return -1;
}
else if ( f1.filtn == 2 ) /* SHUFFLE returns 1 cd_nelmts */
{
if ( f1.cd_nelmts != 1 && f2.cd_nelmts != 0 )
return -1;
}
else
{
if ( f1.cd_nelmts != f2.cd_nelmts )
return -1;
/* consider different filter values as "less" */
for( i = 0; i < f1.cd_nelmts; i++)
{
if (f1.cd_values[i] != f2.cd_values[i])
{
return -1;
}
}
}
/*-------------------------------------------------------------------------
* compare the filter type
*-------------------------------------------------------------------------
*/
if (f1.filtn == f2.filtn)
{
return 0;
}
else if (f1.filtn < f2.filtn)
{
return -1;
}
else if (f1.filtn > f2.filtn)
{
return 1;
}
assert(0);
return -1;
}