mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-12-27 08:01:04 +08:00
Code Issues Packages Projects Releases Wiki Activity
89c9bfc05b
hdf5/tools/h5repack/h5repack_refs.c
Pedro Vicente Nunes bc6ab7c0a6 [svn-r8869] Purpose: 
h5repack changes

Description:
there were some requests to change some minor h5repack features
h5repack only made a warning about a non available filter in verbose mode ( -v )
without -v it kept silent, and users sometimes missed this warning

the request was that it should print this warning always. so, the new format, is e.g

./h5repack -i test_szip.h5 -o out.h5
Warning: dataset </dset_szip> cannot be read, SZIP filter is not available


due to this, and to avoid a lot of these messages in the shell test script, I modified
the script h5repack.sh so that it detects the presence of all filters in the environment
(previously it only detected SZIP)
the test files were also divided in more files , to make the script code easier to
follow


Solution:

Platforms tested:
linux
AIX (no szip)
solaris (no szip, no gzip )

Misc. update:
2004-07-13 14:09:41 -05:00

794 lines
23 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the files COPYING and Copyright.html. COPYING can be found at the root *
* of the source code distribution tree; Copyright.html can be found at the *
* root level of an installed copy of the electronic HDF5 document set and *
* is linked from the top-level documents page. It can also be found at *
* http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have *
* access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "H5private.h"
#include "h5repack.h"
static const char* MapIdToName(hid_t refobj_id,
trav_table_t *travt);
static void close_obj(H5G_obj_t obj_type, hid_t obj_id);
static int copy_refs_attr(hid_t loc_in,
hid_t loc_out,
pack_opt_t *options,
trav_table_t *travt,
hid_t fidout /* for saving references */
);
/*-------------------------------------------------------------------------
* Function: do_copy_refobjs
*
* Purpose: duplicate all referenced HDF5 objects in the file
* and create hard links
*
* Return: 0, ok, -1 no
*
* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
*
* Date: December, 10, 2003
*
*-------------------------------------------------------------------------
*/
int do_copy_refobjs(hid_t fidin,
hid_t fidout,
trav_table_t *travt,
pack_opt_t *options) /* repack options */
{
hid_t grp_in; /* read group ID */
hid_t grp_out; /* write group ID */
hid_t dset_in; /* read dataset ID */
hid_t dset_out; /* write dataset ID */
hid_t type_in; /* named type ID */
hid_t dcpl_id; /* dataset creation property list ID */
hid_t space_id; /* space ID */
hid_t ftype_id; /* file data type ID */
hid_t mtype_id; /* memory data type ID */
size_t msize; /* memory size of memory type */
hsize_t nelmts; /* number of elements in dataset */
int rank; /* rank of dataset */
hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */
int next; /* external files */
int i, j;
/*-------------------------------------------------------------------------
* browse
*-------------------------------------------------------------------------
*/
for ( i = 0; i < travt->nobjs; i++)
{
switch ( travt->objs[i].type )
{
/*-------------------------------------------------------------------------
* H5G_GROUP
*-------------------------------------------------------------------------
*/
case H5G_GROUP:
/*-------------------------------------------------------------------------
* check for hard links
*-------------------------------------------------------------------------
*/
if (travt->objs[i].nlinks)
{
for ( j=0; j<travt->objs[i].nlinks; j++)
{
H5Glink(fidout,
H5G_LINK_HARD,
travt->objs[i].name,
travt->objs[i].links[j].new_name);
}
}
break;
/*-------------------------------------------------------------------------
* H5G_DATASET
*-------------------------------------------------------------------------
*/
case H5G_DATASET:
if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0)
goto error;
if ((space_id=H5Dget_space(dset_in))<0)
goto error;
if ((ftype_id=H5Dget_type (dset_in))<0)
goto error;
if ((dcpl_id=H5Dget_create_plist(dset_in))<0)
goto error;
if ( (rank=H5Sget_simple_extent_ndims(space_id))<0)
goto error;
if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0)
goto error;
nelmts=1;
for (j=0; j<rank; j++)
nelmts*=dims[j];
if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0)
goto error;
if ((msize=H5Tget_size(mtype_id))==0)
goto error;
/*-------------------------------------------------------------------------
* check for external files
*-------------------------------------------------------------------------
*/
if ((next=H5Pget_external_count (dcpl_id))<0)
goto error;
/*-------------------------------------------------------------------------
* check if the dataset creation property list has filters that
* are not registered in the current configuration
* 1) the external filters GZIP and SZIP might not be available
* 2) the internal filters might be turned off
*-------------------------------------------------------------------------
*/
if (next==0 && h5tools_canreadf((NULL),dcpl_id)==1)
{
/*-------------------------------------------------------------------------
* test for a valid output dataset
*-------------------------------------------------------------------------
*/
dset_out = FAIL;
/*-------------------------------------------------------------------------
* object references are a special case
* we cannot just copy the buffers, but instead we recreate the reference
*-------------------------------------------------------------------------
*/
if (H5Tequal(mtype_id, H5T_STD_REF_OBJ))
{
H5G_obj_t obj_type;
hid_t refobj_id;
hobj_ref_t *refbuf=NULL; /* buffer for object references */
hobj_ref_t *buf=NULL;
const char* refname;
unsigned u;
/*-------------------------------------------------------------------------
* read to memory
*-------------------------------------------------------------------------
*/
if (nelmts)
{
buf=(void *) HDmalloc((unsigned)(nelmts*msize));
if ( buf==NULL){
printf( "cannot read into memory\n" );
goto error;
}
if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
goto error;
if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0)
goto error;
refbuf=HDmalloc((unsigned)nelmts*msize);
if ( refbuf==NULL){
printf( "cannot allocate memory\n" );
goto error;
}
for ( u=0; u<nelmts; u++)
{
H5E_BEGIN_TRY {
if ((refobj_id = H5Rdereference(dset_in,H5R_OBJECT,&buf[u]))<0)
continue;
} H5E_END_TRY;
/* get the name. a valid name could only occur in the
second traversal of the file */
if ((refname=MapIdToName(refobj_id,travt))!=NULL)
{
/* create the reference, -1 parameter for objects */
if (H5Rcreate(&refbuf[u],fidout,refname,H5R_OBJECT,-1)<0)
goto error;
if (options->verbose)
printf("object <%s> object reference created to <%s>\n",
travt->objs[i].name,
refname);
}/*refname*/
close_obj(obj_type,refobj_id);
}/* u */
}/*nelmts*/
/*-------------------------------------------------------------------------
* create/write dataset/close
*-------------------------------------------------------------------------
*/
if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0)
goto error;
if (nelmts) {
if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0)
goto error;
}
if (buf)
free(buf);
if (refbuf)
free(refbuf);
}/*H5T_STD_REF_OBJ*/
/*-------------------------------------------------------------------------
* dataset region references
*-------------------------------------------------------------------------
*/
else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG))
{
H5G_obj_t obj_type;
hid_t refobj_id;
hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */
hdset_reg_ref_t *buf=NULL; /* output buffer */
const char* refname;
unsigned u;
/*-------------------------------------------------------------------------
* read input to memory
*-------------------------------------------------------------------------
*/
if (nelmts)
{
buf=(void *) HDmalloc((unsigned)(nelmts*msize));
if ( buf==NULL){
printf( "cannot read into memory\n" );
goto error;
}
if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0)
goto error;
if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0)
goto error;
/*-------------------------------------------------------------------------
* create output
*-------------------------------------------------------------------------
*/
refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */
if ( refbuf==NULL){
printf( "cannot allocate memory\n" );
goto error;
}
for ( u=0; u<nelmts; u++)
{
H5E_BEGIN_TRY {
if ((refobj_id = H5Rdereference(dset_in,H5R_DATASET_REGION,&buf[u]))<0)
continue;
} H5E_END_TRY;
/* get the name. a valid name could only occur in the
second traversal of the file */
if ((refname=MapIdToName(refobj_id,travt))!=NULL)
{
hid_t region_id; /* region id of the referenced dataset */
if ((region_id = H5Rget_region(dset_in,H5R_DATASET_REGION,&buf[u]))<0)
goto error;
/* create the reference, we need the space_id */
if (H5Rcreate(&refbuf[u],fidout,refname,H5R_DATASET_REGION,region_id)<0)
goto error;
if (H5Sclose(region_id)<0)
goto error;
if (options->verbose)
printf("object <%s> region reference created to <%s>\n",
travt->objs[i].name,
refname);
}/*refname*/
close_obj(obj_type,refobj_id);
}/* u */
}/*nelmts*/
/*-------------------------------------------------------------------------
* create/write dataset/close
*-------------------------------------------------------------------------
*/
if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0)
goto error;
if (nelmts) {
if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0)
goto error;
}
if (buf)
free(buf);
if (refbuf)
free(refbuf);
} /* H5T_STD_REF_DSETREG */
/*-------------------------------------------------------------------------
* not references, open previously created object in 1st traversal
*-------------------------------------------------------------------------
*/
else
{
if ((dset_out=H5Dopen(fidout,travt->objs[i].name))<0)
goto error;
}
assert(dset_out!=FAIL);
/*-------------------------------------------------------------------------
* copy referenced objects in attributes
*-------------------------------------------------------------------------
*/
if (copy_refs_attr(dset_in,dset_out,options,travt,fidout)<0)
goto error;
/*-------------------------------------------------------------------------
* check for hard links
*-------------------------------------------------------------------------
*/
if (travt->objs[i].nlinks)
{
for ( j=0; j<travt->objs[i].nlinks; j++){
H5Glink(fidout,
H5G_LINK_HARD,
travt->objs[i].name,
travt->objs[i].links[j].new_name);
}
}
if (H5Dclose(dset_out)<0)
goto error;
}/*can_read*/
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Tclose(ftype_id)<0)
goto error;
if (H5Tclose(mtype_id)<0)
goto error;
if (H5Pclose(dcpl_id)<0)
goto error;
if (H5Sclose(space_id)<0)
goto error;
if (H5Dclose(dset_in)<0)
goto error;
break;
/*-------------------------------------------------------------------------
* H5G_TYPE
*-------------------------------------------------------------------------
*/
case H5G_TYPE:
if ((type_in = H5Topen (fidin,travt->objs[i].name))<0)
goto error;
if (H5Tclose(type_in)<0)
goto error;
break;
/*-------------------------------------------------------------------------
* H5G_LINK
*-------------------------------------------------------------------------
*/
case H5G_LINK:
/*nothing to do */
break;
default:
break;
}
}
/*-------------------------------------------------------------------------
* the root is a special case, we get an ID for the root group
* and copy its attributes using that ID
* it must be done last, because the attributes might contain references to
* objects in the object list
*-------------------------------------------------------------------------
*/
if ((grp_out = H5Gopen(fidout,"/"))<0)
goto error;
if ((grp_in = H5Gopen(fidin,"/"))<0)
goto error;
if (copy_refs_attr(grp_in,grp_out,options,travt,fidout)<0)
goto error;
if (H5Gclose(grp_out)<0)
goto error;
if (H5Gclose(grp_in)<0)
goto error;
return 0;
error:
H5E_BEGIN_TRY {
H5Gclose(grp_in);
H5Gclose(grp_out);
H5Pclose(dcpl_id);
H5Sclose(space_id);
H5Dclose(dset_in);
H5Dclose(dset_out);
H5Tclose(ftype_id);
H5Tclose(mtype_id);
H5Tclose(type_in);
} H5E_END_TRY;
return -1;
}
/*-------------------------------------------------------------------------
* Function: copy_refs_attr
*
* Purpose: duplicate all referenced HDF5 located in attributes
* relative to LOC_IN, which is obtained either from
* loc_id = H5Gopen( fid, name);
* loc_id = H5Dopen( fid, name);
* loc_id = H5Topen( fid, name);
*
* Return: 0, ok, -1 no
*
* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
*
* Date: October, 28, 2003
*
*-------------------------------------------------------------------------
*/
static int copy_refs_attr(hid_t loc_in,
hid_t loc_out,
pack_opt_t *options,
trav_table_t *travt,
hid_t fidout /* for saving references */
)
{
hid_t attr_id=-1; /* attr ID */
hid_t attr_out=-1; /* attr ID */
hid_t space_id=-1; /* space ID */
hid_t ftype_id=-1; /* file data type ID */
hid_t mtype_id=-1; /* memory data type ID */
size_t msize; /* memory size of type */
hsize_t nelmts; /* number of elements in dataset */
int rank; /* rank of dataset */
hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */
char name[255];
int n, j;
unsigned u;
if ((n = H5Aget_num_attrs(loc_in))<0)
goto error;
for ( u = 0; u < (unsigned)n; u++)
{
/*-------------------------------------------------------------------------
* open
*-------------------------------------------------------------------------
*/
/* open attribute */
if ((attr_id = H5Aopen_idx(loc_in, u))<0)
goto error;
/* get name */
if (H5Aget_name( attr_id, 255, name )<0)
goto error;
/* get the file datatype */
if ((ftype_id = H5Aget_type( attr_id )) < 0 )
goto error;
/* get the dataspace handle */
if ((space_id = H5Aget_space( attr_id )) < 0 )
goto error;
/* get dimensions */
if ( (rank = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0 )
goto error;
/*-------------------------------------------------------------------------
* elements
*-------------------------------------------------------------------------
*/
nelmts=1;
for (j=0; j<rank; j++)
nelmts*=dims[j];
if ((mtype_id=H5Tget_native_type(ftype_id,H5T_DIR_DEFAULT))<0)
goto error;
if ((msize=H5Tget_size(mtype_id))==0)
goto error;
/*-------------------------------------------------------------------------
* object references are a special case
* we cannot just copy the buffers, but instead we recreate the reference
*-------------------------------------------------------------------------
*/
if (H5Tequal(mtype_id, H5T_STD_REF_OBJ))
{
H5G_obj_t obj_type;
hid_t refobj_id;
hobj_ref_t *refbuf=NULL;
unsigned k;
const char* refname;
hobj_ref_t *buf=NULL;
/*-------------------------------------------------------------------------
* read input to memory
*-------------------------------------------------------------------------
*/
if (nelmts)
{
buf=(void *) HDmalloc((unsigned)(nelmts*msize));
if ( buf==NULL){
printf( "cannot read into memory\n" );
goto error;
}
if (H5Aread(attr_id,mtype_id,buf)<0)
goto error;
if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0)
goto error;
refbuf=HDmalloc((unsigned)nelmts*msize);
if ( refbuf==NULL){
printf( "cannot allocate memory\n" );
goto error;
}
for ( k=0; k<nelmts; k++)
{
H5E_BEGIN_TRY {
if ((refobj_id = H5Rdereference(attr_id,H5R_OBJECT,&buf[k]))<0)
goto error;
} H5E_END_TRY;
/* get the name. a valid name could only occur in the
second traversal of the file */
if ((refname=MapIdToName(refobj_id,travt))!=NULL)
{
/* create the reference */
if (H5Rcreate(&refbuf[k],fidout,refname,H5R_OBJECT,-1)<0)
goto error;
if (options->verbose)
printf("object <%s> reference created to <%s>\n",name,refname);
}
close_obj(obj_type,refobj_id);
}/* k */
}/*nelmts*/
/*-------------------------------------------------------------------------
* copy
*-------------------------------------------------------------------------
*/
if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0)
goto error;
if (nelmts) {
if(H5Awrite(attr_out,mtype_id,refbuf)<0)
goto error;
}
if (H5Aclose(attr_out)<0)
goto error;
if (refbuf)
free(refbuf);
if (buf)
free(buf);
}/*H5T_STD_REF_OBJ*/
/*-------------------------------------------------------------------------
* dataset region references
*-------------------------------------------------------------------------
*/
else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG))
{
H5G_obj_t obj_type;
hid_t refobj_id;
hdset_reg_ref_t *refbuf=NULL; /* input buffer for region references */
hdset_reg_ref_t *buf=NULL; /* output buffer */
const char* refname;
unsigned k;
/*-------------------------------------------------------------------------
* read input to memory
*-------------------------------------------------------------------------
*/
if (nelmts)
{
buf=(void *) HDmalloc((unsigned)(nelmts*msize));
if ( buf==NULL){
printf( "cannot read into memory\n" );
goto error;
}
if (H5Aread(attr_id,mtype_id,buf)<0)
goto error;
if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0)
goto error;
/*-------------------------------------------------------------------------
* create output
*-------------------------------------------------------------------------
*/
refbuf=HDcalloc(sizeof(hdset_reg_ref_t),(size_t)nelmts); /*init to zero */
if ( refbuf==NULL){
printf( "cannot allocate memory\n" );
goto error;
}
for ( k=0; k<nelmts; k++)
{
H5E_BEGIN_TRY {
if ((refobj_id = H5Rdereference(attr_id,H5R_DATASET_REGION,&buf[k]))<0)
continue;
} H5E_END_TRY;
/* get the name. a valid name could only occur in the
second traversal of the file */
if ((refname=MapIdToName(refobj_id,travt))!=NULL)
{
hid_t region_id; /* region id of the referenced dataset */
if ((region_id = H5Rget_region(attr_id,H5R_DATASET_REGION,&buf[k]))<0)
goto error;
/* create the reference, we need the space_id */
if (H5Rcreate(&refbuf[k],fidout,refname,H5R_DATASET_REGION,region_id)<0)
goto error;
if (H5Sclose(region_id)<0)
goto error;
if (options->verbose)
printf("object <%s> region reference created to <%s>\n",name,refname);
}
close_obj(obj_type,refobj_id);
}/* k */
}/*nelmts */
/*-------------------------------------------------------------------------
* copy
*-------------------------------------------------------------------------
*/
if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0)
goto error;
if (nelmts) {
if(H5Awrite(attr_out,mtype_id,refbuf)<0)
goto error;
}
if (H5Aclose(attr_out)<0)
goto error;
if (refbuf)
free(refbuf);
if (buf)
free(buf);
} /* H5T_STD_REF_DSETREG */
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
if (H5Tclose(ftype_id)<0) goto error;
if (H5Tclose(mtype_id)<0) goto error;
if (H5Sclose(space_id)<0) goto error;
if (H5Aclose(attr_id)<0) goto error;
} /* u */
return 0;
error:
H5E_BEGIN_TRY {
H5Tclose(ftype_id);
H5Tclose(mtype_id);
H5Sclose(space_id);
H5Aclose(attr_id);
H5Aclose(attr_out);
} H5E_END_TRY;
return -1;
}
/*-------------------------------------------------------------------------
* Function: close_obj
*
* Purpose: Auxiliary function to close an object
*
*-------------------------------------------------------------------------
*/
static void close_obj(H5G_obj_t obj_type, hid_t obj_id)
{
H5E_BEGIN_TRY
{
switch (obj_type)
{
case H5G_GROUP:
H5Gclose(obj_id);
break;
case H5G_DATASET:
H5Dclose(obj_id);
break;
case H5G_TYPE:
H5Tclose(obj_id);
break;
default:
break;
}
} H5E_END_TRY;
}
/*-------------------------------------------------------------------------
* Function: MapIdToName
*
* Purpose: map an object ID to a name
*
*-------------------------------------------------------------------------
*/
static const char* MapIdToName(hid_t refobj_id,
trav_table_t *travt)
{
hid_t id;
hid_t fid;
int i;
/* obtains the file ID given an object ID. This ID must be closed */
if ((fid = H5Iget_file_id(refobj_id))<0)
{
return NULL;
}
/* linear search */
for ( i=0; i<travt->nobjs; i++)
{
switch ( travt->objs[i].type )
{
default:
break;
/*-------------------------------------------------------------------------
* H5G_DATASET
*-------------------------------------------------------------------------
*/
case H5G_DATASET:
if ((id = H5Dopen(fid,travt->objs[i].name))<0)
return NULL;
if (H5Dclose(id)<0)
return NULL;
if (id==refobj_id)
{
H5Fclose(fid);
return travt->objs[i].name;
}
break;
} /* switch */
} /* i */
if (H5Fclose(fid)<0)
return NULL;
return NULL;
}
Reference in New Issue View Git Blame Copy Permalink