H5R: fix type conversion of references within vlen type

Update trefer.c with vlen test case
This commit is contained in:
Jerome Soumagne 2020-07-14 20:00:47 -05:00
parent 806f5b5b2a
commit aa9d2bd9bc
4 changed files with 336 additions and 25 deletions

View File

@ -423,6 +423,10 @@ H5R__destroy(H5R_ref_priv_t *ref)
HDassert(ref != NULL);
H5R_LOG_DEBUG("Destroying reference, filename=%s, obj_addr=%s, encode size=%u",
ref->info.obj.filename, H5R__print_token(ref->info.obj.token),
ref->encode_size);
H5MM_xfree(ref->info.obj.filename);
ref->info.obj.filename = NULL;
@ -1028,6 +1032,10 @@ H5R__encode(const char *filename, const H5R_ref_priv_t *ref, unsigned char *buf,
HGOTO_ERROR(H5E_REFERENCE, H5E_UNSUPPORTED, FAIL, "internal error (unknown reference type)")
} /* end switch */
H5R_LOG_DEBUG("Encoded reference, filename=%s, obj_addr=%s, encode size=%u",
ref->info.obj.filename, H5R__print_token(ref->info.obj.token),
encode_size);
*nalloc = encode_size;
done:

View File

@ -3110,8 +3110,24 @@ H5T__conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts,
if(NULL == (tpath = H5T_path_find(src->shared->parent, dst->shared->parent)))
HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest datatypes")
else if(!H5T_path_noop(tpath)) {
if((tsrc_id = H5I_register(H5I_DATATYPE, H5T_copy(src->shared->parent, H5T_COPY_ALL), FALSE)) < 0 ||
(tdst_id = H5I_register(H5I_DATATYPE, H5T_copy(dst->shared->parent, H5T_COPY_ALL), FALSE)) < 0)
H5T_t *tsrc_cpy = NULL, *tdst_cpy = NULL;
if(NULL == (tsrc_cpy = H5T_copy(src->shared->parent, H5T_COPY_ALL)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy src type for conversion")
/* References need to know about the src file */
if(tsrc_cpy->shared->type == H5T_REFERENCE)
if(H5T_set_loc(tsrc_cpy, src->shared->u.vlen.file, H5T_LOC_MEMORY) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set datatype location");
if(NULL == (tdst_cpy = H5T_copy(dst->shared->parent, H5T_COPY_ALL)))
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy dst type for conversion")
/* References need to know about the dst file */
if(tdst_cpy->shared->type == H5T_REFERENCE)
if(H5T_set_loc(tdst_cpy, dst->shared->u.vlen.file, H5T_LOC_MEMORY) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set datatype location");
if(((tsrc_id = H5I_register(H5I_DATATYPE, tsrc_cpy, FALSE)) < 0)
|| ((tdst_id = H5I_register(H5I_DATATYPE, tdst_cpy, FALSE)) < 0))
HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register types for conversion")
} /* end else-if */
else

View File

@ -144,7 +144,9 @@ H5T__ref_set_loc(const H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc)
switch(loc) {
case H5T_LOC_MEMORY: /* Memory based reference datatype */
HDassert(NULL == file);
/* NB. We allow for the file to be non-NULL when doing
* memory-to-memory conversion */
/* Mark this type as being stored in memory */
dt->shared->u.atomic.u.r.loc = H5T_LOC_MEMORY;
@ -364,9 +366,8 @@ static size_t
H5T__ref_mem_getsize(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
size_t H5_ATTR_UNUSED src_size, H5VL_object_t *dst_file, hbool_t *dst_copy)
{
H5VL_object_t *vol_obj; /* VOL object for src ref's location */
H5VL_object_t *vol_obj = NULL; /* VOL object for src ref's location */
const H5R_ref_priv_t *src_ref = (const H5R_ref_priv_t *)src_buf;
hbool_t files_equal = FALSE; /* Whether src & dst references are in same file */
char *file_name_buf_dyn = NULL; /* Pointer to dynamically allocated buffer for file name, if static buffer is too small */
unsigned flags = 0; /* References flags */
size_t ret_value = 0; /* Return value */
@ -377,14 +378,18 @@ H5T__ref_mem_getsize(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf
HDassert(src_buf);
HDassert(src_size == H5T_REF_MEM_SIZE);
/* Retrieve VOL object */
if(NULL == (vol_obj = H5VL_vol_object(src_ref->loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "invalid location identifier")
if(NULL != dst_file) {
hbool_t files_equal = TRUE; /* Whether src & dst references are in same file */
/* Set external flag if referenced file is not destination file */
if(H5VL_file_is_same(vol_obj, dst_file, &files_equal) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTCOMPARE, 0, "can't check if files are equal")
flags |= !files_equal ? H5R_IS_EXTERNAL : 0;
/* Retrieve VOL object */
if(NULL == (vol_obj = H5VL_vol_object(src_ref->loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "invalid location identifier")
/* Set external flag if referenced file is not destination file */
if(H5VL_file_is_same(vol_obj, dst_file, &files_equal) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTCOMPARE, 0, "can't check if files are equal")
flags |= !files_equal ? H5R_IS_EXTERNAL : 0;
}
/* Force re-calculating encoding size if any flags are set */
if(flags || !src_ref->encode_size) {
@ -458,8 +463,8 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
{
H5VL_object_t *vol_obj; /* VOL object for src ref's location */
const H5R_ref_priv_t *src_ref = (const H5R_ref_priv_t *)src_buf;
hbool_t files_equal = FALSE; /* Whether src & dst references are in same file */
char file_name_buf_static[256]; /* File name */
hbool_t files_equal = TRUE; /* Whether src & dst references are in same file */
char file_name_buf_static[256] = {'\0'}; /* File name */
char *file_name_buf_dyn = NULL; /* Pointer to dynamically allocated buffer for file name, if static buffer is too small */
ssize_t file_name_len; /* Size of file name buffer */
unsigned flags = 0; /* References flags */
@ -470,10 +475,15 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
/* Sanity check */
HDassert(src_buf);
HDassert(src_size == H5T_REF_MEM_SIZE);
HDassert(dst_file);
HDassert(dst_buf);
HDassert(dst_size);
/* Memory-to-memory conversion to support vlen conversion */
if(NULL == dst_file) {
HDmemcpy(dst_buf, src_buf, dst_size);
HGOTO_DONE(ret_value);
}
/* Retrieve VOL object */
if(NULL == (vol_obj = H5VL_vol_object(src_ref->loc_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "invalid location identifier")
@ -504,14 +514,16 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf,
H5CX_set_libver_bounds(NULL);
} /* end if */
/* Get file name */
if(H5VL_file_get(vol_obj, H5VL_FILE_GET_NAME, H5P_DATASET_XFER_DEFAULT, NULL, H5I_FILE, sizeof(file_name_buf_static), file_name_buf_static, &file_name_len) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, 0, "can't get file name")
if(file_name_len >= (ssize_t)sizeof(file_name_buf_static)) {
if(NULL == (file_name_buf_dyn = (char *)H5MM_malloc((size_t)file_name_len + 1)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, "can't allocate space for file name")
if(H5VL_file_get(vol_obj, H5VL_FILE_GET_NAME, H5P_DATASET_XFER_DEFAULT, NULL, H5I_FILE, (size_t)file_name_len + 1, file_name_buf_dyn, &file_name_len) < 0)
/* Get file name (if external reference) */
if(flags) {
if(H5VL_file_get(vol_obj, H5VL_FILE_GET_NAME, H5P_DATASET_XFER_DEFAULT, NULL, H5I_FILE, sizeof(file_name_buf_static), file_name_buf_static, &file_name_len) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, 0, "can't get file name")
if(file_name_len >= (ssize_t)sizeof(file_name_buf_static)) {
if(NULL == (file_name_buf_dyn = (char *)H5MM_malloc((size_t)file_name_len + 1)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, "can't allocate space for file name")
if(H5VL_file_get(vol_obj, H5VL_FILE_GET_NAME, H5P_DATASET_XFER_DEFAULT, NULL, H5I_FILE, (size_t)file_name_len + 1, file_name_buf_dyn, &file_name_len) < 0)
HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, 0, "can't get file name")
} /* end if */
} /* end if */
/* Encode reference */
@ -539,7 +551,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size
H5R_type_t src_type, H5VL_object_t H5_ATTR_UNUSED *dst_file, void *dst_buf,
size_t dst_size, void H5_ATTR_UNUSED *bg_buf)
{
H5F_t *src_f;
H5F_t *src_f = NULL;
hid_t file_id = H5I_INVALID_HID;
H5R_ref_priv_t *dst_ref = (H5R_ref_priv_t *)dst_buf;
herr_t ret_value = SUCCEED;
@ -547,14 +559,19 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size
FUNC_ENTER_STATIC
/* Sanity check */
HDassert(src_file);
HDassert(src_buf);
HDassert(src_size);
HDassert(dst_buf);
HDassert(dst_size == H5T_REF_MEM_SIZE);
/* Memory-to-memory conversion to support vlen conversion */
if(NULL == src_file) {
HDmemcpy(dst_buf, src_buf, src_size);
HGOTO_DONE(ret_value);
}
#ifndef NDEBUG
{
if((src_type == H5R_OBJECT1) || (src_type == H5R_DATASET_REGION1)) {
hbool_t is_native = FALSE; /* Whether the src file is using the native VOL connector */
/* Check if using native VOL connector */

View File

@ -23,6 +23,7 @@
#define FILE_REF_PARAM "trefer_param.h5"
#define FILE_REF_OBJ "trefer_obj.h5"
#define FILE_REF_VL_OBJ "trefer_vl_obj.h5"
#define FILE_REF_REG "trefer_reg.h5"
#define FILE_REF_REG_1D "trefer_reg_1d.h5"
#define FILE_REF_OBJ_DEL "trefer_obj_del.h5"
@ -536,6 +537,273 @@ test_reference_obj(void)
HDfree(obuf);
} /* test_reference_obj() */
/****************************************************************
**
** test_reference_vlen_obj(): Test basic H5R (reference) object reference
** within a vlen type.
** Tests references to various kinds of objects
**
****************************************************************/
static void
test_reference_vlen_obj(void)
{
hid_t fid1; /* HDF5 File IDs */
hid_t dataset, /* Dataset ID */
dset2; /* Dereferenced dataset ID */
hid_t group; /* Group ID */
hid_t sid1; /* Dataspace ID */
hid_t tid1; /* Datatype ID */
hsize_t dims1[] = {SPACE1_DIM1};
hsize_t vl_dims[] = {1};
hid_t dapl_id; /* Dataset access property list */
H5R_ref_t *wbuf, /* buffer to write to disk */
*rbuf; /* buffer read from disk */
unsigned *ibuf, *obuf;
unsigned i, j; /* Counters */
H5O_type_t obj_type; /* Object type */
herr_t ret; /* Generic return value */
hvl_t vl_wbuf = {0, NULL}, vl_rbuf = {0, NULL};
/* Output message about test being performed */
MESSAGE(5, ("Testing Object Reference Functions within VLEN type\n"));
/* Allocate write & read buffers */
wbuf = HDcalloc(sizeof(H5R_ref_t), SPACE1_DIM1);
rbuf = HDcalloc(sizeof(H5R_ref_t), SPACE1_DIM1);
ibuf = HDcalloc(sizeof(unsigned), SPACE1_DIM1);
obuf = HDcalloc(sizeof(unsigned), SPACE1_DIM1);
for (i = 0; i < SPACE1_DIM1; i++)
obuf[i] = i * 3;
/* Create file */
fid1 = H5Fcreate(FILE_REF_VL_OBJ, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
CHECK(fid1, H5I_INVALID_HID, "H5Fcreate");
/* Create dataspace for datasets */
sid1 = H5Screate_simple(SPACE1_RANK, dims1, NULL);
CHECK(sid1, H5I_INVALID_HID, "H5Screate_simple");
/* Create dataset access property list */
dapl_id = H5Pcreate(H5P_DATASET_ACCESS);
CHECK(dapl_id, H5I_INVALID_HID, "H5Pcreate");
/* Create a group */
group = H5Gcreate2(fid1, "Group1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(group, H5I_INVALID_HID, "H5Gcreate2");
/* Create a dataset (inside Group1) */
dataset = H5Dcreate2(group, "Dataset1", H5T_NATIVE_UINT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(dataset, H5I_INVALID_HID, "H5Dcreate2");
/* Write selection to disk */
ret = H5Dwrite(dataset, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, obuf);
CHECK(ret, FAIL, "H5Dwrite");
/* Close Dataset */
ret = H5Dclose(dataset);
CHECK(ret, FAIL, "H5Dclose");
/* Create another dataset (inside Group1) */
dataset = H5Dcreate2(group, "Dataset2", H5T_NATIVE_UCHAR, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(dataset, FAIL, "H5Dcreate2");
/* Close Dataset */
ret = H5Dclose(dataset);
CHECK(ret, FAIL, "H5Dclose");
/* Close disk dataspace */
ret = H5Sclose(sid1);
CHECK(ret, FAIL, "H5Sclose");
/* Create a datatype to refer to */
tid1 = H5Tcreate(H5T_COMPOUND, sizeof(s1_t));
CHECK(tid1, H5I_INVALID_HID, "H5Tcreate");
/* Insert fields */
ret = H5Tinsert(tid1, "a", HOFFSET(s1_t,a), H5T_NATIVE_INT);
CHECK(ret, FAIL, "H5Tinsert");
ret = H5Tinsert(tid1, "b", HOFFSET(s1_t,b), H5T_NATIVE_INT);
CHECK(ret, FAIL, "H5Tinsert");
ret = H5Tinsert(tid1, "c", HOFFSET(s1_t,c), H5T_NATIVE_FLOAT);
CHECK(ret, FAIL, "H5Tinsert");
/* Save datatype for later */
ret = H5Tcommit2(group, "Datatype1", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(ret, FAIL, "H5Tcommit2");
/* Close datatype */
ret = H5Tclose(tid1);
CHECK(ret, FAIL, "H5Tclose");
/* Close group */
ret = H5Gclose(group);
CHECK(ret, FAIL, "H5Gclose");
/* Create vlen type */
tid1 = H5Tvlen_create(H5T_STD_REF);
CHECK(tid1, H5I_INVALID_HID, "H5Tvlen_create");
/* Create dataspace for datasets */
sid1 = H5Screate_simple(SPACE1_RANK, vl_dims, NULL);
CHECK(sid1, H5I_INVALID_HID, "H5Screate_simple");
/* Create a dataset */
dataset = H5Dcreate2(fid1, "Dataset3", tid1, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
CHECK(dataset, H5I_INVALID_HID, "H5Dcreate2");
/* Create reference to dataset */
ret = H5Rcreate_object(fid1, "/Group1/Dataset1", H5P_DEFAULT, &wbuf[0]);
CHECK(ret, FAIL, "H5Rcreate_object");
ret = H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type);
CHECK(ret, FAIL, "H5Rget_obj_type3");
VERIFY(obj_type, H5O_TYPE_DATASET, "H5Rget_obj_type3");
/* Create reference to dataset */
ret = H5Rcreate_object(fid1, "/Group1/Dataset2", H5P_DEFAULT, &wbuf[1]);
CHECK(ret, FAIL, "H5Rcreate_object");
ret = H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type);
CHECK(ret, FAIL, "H5Rget_obj_type3");
VERIFY(obj_type, H5O_TYPE_DATASET, "H5Rget_obj_type3");
/* Create reference to group */
ret = H5Rcreate_object(fid1, "/Group1", H5P_DEFAULT, &wbuf[2]);
CHECK(ret, FAIL, "H5Rcreate_object");
ret = H5Rget_obj_type3(&wbuf[2], H5P_DEFAULT, &obj_type);
CHECK(ret, FAIL, "H5Rget_obj_type3");
VERIFY(obj_type, H5O_TYPE_GROUP, "H5Rget_obj_type3");
/* Create reference to named datatype */
ret = H5Rcreate_object(fid1, "/Group1/Datatype1", H5P_DEFAULT, &wbuf[3]);
CHECK(ret, FAIL, "H5Rcreate_object");
ret = H5Rget_obj_type3(&wbuf[3], H5P_DEFAULT, &obj_type);
CHECK(ret, FAIL, "H5Rget_obj_type3");
VERIFY(obj_type, H5O_TYPE_NAMED_DATATYPE, "H5Rget_obj_type3");
/* Store references into vlen */
vl_wbuf.len = SPACE1_DIM1;
vl_wbuf.p = wbuf;
/* Write selection to disk */
ret = H5Dwrite(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vl_wbuf);
CHECK(ret, FAIL, "H5Dwrite");
/* Close disk dataspace */
ret = H5Sclose(sid1);
CHECK(ret, FAIL, "H5Sclose");
/* Close Dataset */
ret = H5Dclose(dataset);
CHECK(ret, FAIL, "H5Dclose");
/* Close datatype */
ret = H5Tclose(tid1);
CHECK(ret, FAIL, "H5Tclose");
/* Close file */
ret = H5Fclose(fid1);
CHECK(ret, FAIL, "H5Fclose");
/* Re-open the file */
fid1 = H5Fopen(FILE_REF_VL_OBJ, H5F_ACC_RDWR, H5P_DEFAULT);
CHECK(fid1, H5I_INVALID_HID, "H5Fopen");
/* Open the dataset */
dataset = H5Dopen2(fid1, "/Dataset3", H5P_DEFAULT);
CHECK(dataset, H5I_INVALID_HID, "H5Dopen2");
tid1 = H5Dget_type(dataset);
CHECK(tid1, H5I_INVALID_HID, "H5Dget_type");
/* Read selection from disk */
ret = H5Dread(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vl_rbuf);
CHECK(ret, FAIL, "H5Dread");
VERIFY(vl_rbuf.len, SPACE1_DIM1, "H5Dread");
rbuf = vl_rbuf.p;
/* Close datatype */
ret = H5Tclose(tid1);
CHECK(ret, FAIL, "H5Tclose");
/* Open dataset object */
dset2 = H5Ropen_object(&rbuf[0], H5P_DEFAULT, dapl_id);
CHECK(dset2, H5I_INVALID_HID, "H5Ropen_object");
/* Check information in referenced dataset */
sid1 = H5Dget_space(dset2);
CHECK(sid1, H5I_INVALID_HID, "H5Dget_space");
ret = (int)H5Sget_simple_extent_npoints(sid1);
VERIFY(ret, SPACE1_DIM1, "H5Sget_simple_extent_npoints");
/* Read from disk */
ret = H5Dread(dset2, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf);
CHECK(ret, FAIL, "H5Dread");
for(i = 0; i < SPACE1_DIM1; i++)
VERIFY(ibuf[i], i * 3, "Data");
/* Close dereferenced Dataset */
ret = H5Dclose(dset2);
CHECK(ret, FAIL, "H5Dclose");
/* Open group object. GAPL isn't supported yet. But it's harmless to pass in */
group = H5Ropen_object(&rbuf[2], H5P_DEFAULT, H5P_DEFAULT);
CHECK(group, H5I_INVALID_HID, "H5Ropen_object");
/* Close group */
ret = H5Gclose(group);
CHECK(ret, FAIL, "H5Gclose");
/* Open datatype object. TAPL isn't supported yet. But it's harmless to pass in */
tid1 = H5Ropen_object(&rbuf[3], H5P_DEFAULT, H5P_DEFAULT);
CHECK(tid1, H5I_INVALID_HID, "H5Ropen_object");
/* Verify correct datatype */
{
H5T_class_t tclass;
tclass = H5Tget_class(tid1);
VERIFY(tclass, H5T_COMPOUND, "H5Tget_class");
ret= H5Tget_nmembers(tid1);
VERIFY(ret, 3, "H5Tget_nmembers");
}
/* Close datatype */
ret = H5Tclose(tid1);
CHECK(ret, FAIL, "H5Tclose");
/* Close Dataset */
ret = H5Dclose(dataset);
CHECK(ret, FAIL, "H5Dclose");
/* Close dataset access property list */
ret = H5Pclose(dapl_id);
CHECK(ret, FAIL, "H5Pclose");
/* Close file */
ret = H5Fclose(fid1);
CHECK(ret, FAIL, "H5Fclose");
/* Destroy references */
for(j = 0; j < SPACE1_DIM1; j++) {
ret = H5Rdestroy(&wbuf[j]);
CHECK(ret, FAIL, "H5Rdestroy");
ret = H5Rdestroy(&rbuf[j]);
CHECK(ret, FAIL, "H5Rdestroy");
}
/* Free memory buffers */
HDfree(wbuf);
HDfree(rbuf);
HDfree(ibuf);
HDfree(obuf);
} /* test_reference_vlen_obj() */
/****************************************************************
**
** test_reference_region(): Test basic H5R (reference) object reference code.
@ -2862,6 +3130,7 @@ test_reference(void)
test_reference_params(); /* Test for correct parameter checking */
test_reference_obj(); /* Test basic H5R object reference code */
test_reference_vlen_obj(); /* Test reference within vlen */
/* Loop through all the combinations of low/high version bounds */
for(low = H5F_LIBVER_EARLIEST; low < H5F_LIBVER_NBOUNDS; low++) {
@ -2907,6 +3176,7 @@ cleanup_reference(void)
{
HDremove(FILE_REF_PARAM);
HDremove(FILE_REF_OBJ);
HDremove(FILE_REF_VL_OBJ);
HDremove(FILE_REF_REG);
HDremove(FILE_REF_REG_1D);
HDremove(FILE_REF_OBJ_DEL);