mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-31 17:10:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Onion VFD (#1953)

Browse Source 
* Onion VFD feature

* Fixes onion VFD errors with non-sec2 backing store VFDs

* Disables the onion VFD tests w/ ph5diff

* Disables non-sec2 VFDs as onion VFD backing stores

* Committing clang-format changes

* Formatted source

* Typo

* Adds onion VFD tools tests to CMake

* Fixes for v16 API compatibility

* Memset structs to avoid bad frees on errors

* H5Dwrite() calls now use H5T_NATIVE_INT as the memory type vs LE

* Properly decodes checksums on BE machines

* Be more careful about uint64_t to haddr_t/hsize_t conversions

* Another fix for BE data comparison

* Removed double underscores from onion constants

* Replace hard-coded onion header string w/ constant

* Fixes cleanup paths in H5FD__onion_ingest_history()

* Fixed use of size_t revision numbers

* Fix h5dump revision count format string

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Dana Robinson 2022-08-02 12:54:40 -07:00 committed by GitHub
parent ea13de1bb0
commit fcf41b3cd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
55 changed files with 10632 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/trace
View File

@ -200,6 +200,7 @@ $Source = "";
"H5FD_t" => "#",
"H5FD_hdfs_fapl_t" => "#",
"H5FD_mirror_fapl_t" => "#",
"H5FD_onion_fapl_t" => "#",
"H5FD_ros3_fapl_t" => "#",
"H5FD_splitter_vfd_config_t" => "#",
"H5L_class_t" => "#",

9
src/CMakeLists.txt
View File

@ -240,6 +240,10 @@ set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FDmpi.c
${HDF5_SRC_DIR}/H5FDmpio.c
${HDF5_SRC_DIR}/H5FDmulti.c
${HDF5_SRC_DIR}/H5FDonion.c
${HDF5_SRC_DIR}/H5FDonion_header.c
${HDF5_SRC_DIR}/H5FDonion_history.c
${HDF5_SRC_DIR}/H5FDonion_index.c
${HDF5_SRC_DIR}/H5FDperform.c
${HDF5_SRC_DIR}/H5FDros3.c
${HDF5_SRC_DIR}/H5FDs3comms.c
@ -262,6 +266,7 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDmpi.h
${HDF5_SRC_DIR}/H5FDmpio.h
${HDF5_SRC_DIR}/H5FDmulti.h
${HDF5_SRC_DIR}/H5FDonion.h
${HDF5_SRC_DIR}/H5FDpublic.h
${HDF5_SRC_DIR}/H5FDros3.h
${HDF5_SRC_DIR}/H5FDs3comms.h
@ -902,6 +907,10 @@ set (H5_PRIVATE_HEADERS
${HDF5_SRC_DIR}/H5FAprivate.h
${HDF5_SRC_DIR}/H5FDmirror_priv.h
${HDF5_SRC_DIR}/H5FDonion_header.h
${HDF5_SRC_DIR}/H5FDonion_history.h
${HDF5_SRC_DIR}/H5FDonion_index.h
${HDF5_SRC_DIR}/H5FDonion_priv.h
${HDF5_SRC_DIR}/H5FDpkg.h
${HDF5_SRC_DIR}/H5FDprivate.h

1762
src/H5FDonion.c Normal file
View File

File diff suppressed because it is too large Load Diff

200
src/H5FDonion.h Normal file
View File

@ -0,0 +1,200 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: The public header file for the Onion VFD.
*/
#ifndef H5FDonion_H
#define H5FDonion_H
#define H5FD_ONION (H5FDperform_init(H5FD_onion_init))
#define H5FD_ONION_VALUE H5_VFD_ONION
/* Current version of the fapl info struct */
#define H5FD_ONION_FAPL_INFO_VERSION_CURR 1
/* Flag to open a file that has a locked header (after crashes, for example) */
#define H5FD_ONION_FAPL_INFO_FLAG_FORCE_OPEN 1
/* Flag to enable opening older revisions in write mode, creating a tree */
#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY 0x1
/* Flag to require page alignment of onion revision data */
#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT 0x2
/* Max length of a comment */
#define H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN 255
/* Indicates that you want the latest revision
* TODO: Does this work?
*/
#define H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST UINT64_MAX
typedef enum H5FD_onion_target_file_constant_t {
H5FD_ONION_STORE_TARGET_H5, /* Onion history as part of HDF5 file */
H5FD_ONION_STORE_TARGET_ONION, /* Separate, single "onion" file */
} H5FD_onion_target_file_constant_t;
/*-----------------------------------------------------------------------------
* Structure H5FD_onion_fapl_info_t
*
* Purpose: Encapsulate info for the Onion driver FAPL entry.
*
* version: Future-proofing identifier. Informs struct membership.
* Must equal H5FD_ONION_FAPL_VERSION_CURR to be considered valid.
*
* backing_fapl_id:
* Backing or 'child' FAPL ID to handle I/O with the
* underlying backing store. If the onion data is stored as a
* separate file, it must use the same backing driver as the
* original file.
*
* page_size: Size of the amended data pages. If opening an existing file,
* must equal the existing page size or zero. If creating a new
* file or an initial revision of an existing file, must be a
* power of 2.
*
* store_target:
* Enumerated/defined value identifying where the history data is
* stored, either in the same file (appended to HDF5 data) or a
* separate file. Other options may be added in later versions.
*
* + H5FD_ONION_FAPL_STORE_MODE_SEPARATE_SINGLE (1)
* Onion history is stored in a single, separate "onion
* file". Shares filename and path as hdf5 file (if any),
* with only a different filename extension.
*
* revision_num: Which revision to open. Must be 0 (the original file) or the
* revision number of an existing revision.
* Revision ID -1 is reserved to open the most recently-created
* revision in history.
*
* force_write_open:
* Flag to ignore the write-lock flag in the onion data
* and attempt to open the file write-only anyway.
* This may be relevant if, for example, the library crashed
* while the file was open in write mode and the write-lock
* flag was not cleared.
* Must equal H5FD_ONION_FAPL_FLAG_FORCE_OPEN to enable.
*
* creation_flags:
* Flag used only when instantiating an Onion file.
* If the relevant bit is set to a nonzero value, its feature
* will be enabled.
*
* + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY
* (1, bit 1)
* User will be allowed to open arbitrary revisions
* in write mode.
* If disabled (0), only the most recent revision may be
* opened for amendment.
*
* + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT (2, bit 2)
* Onion history metadata will align to page_size.
* Partial pages of unused space will occur in the file,
* but may improve read performance from the backing store
* on some systems.
* If disabled (0), padding will not be inserted to align
* to page boundaries.
*
* + <Remaining bits reserved>
*
* comment: User-supplied NULL-terminated comment for a revision to be
* written.
* Cannot be longer than H5FD_ONION_FAPL_COMMENT_MAX_LEN.
* Ignored if part of a FAPL used to open in read mode.
*
* The comment for a revision may be modified prior to committing
* to the revision (closing the file and writing the record)
* with a call to H5FDfctl().
* This H5FDfctl overwrite may be used to exceed constraints of
* maximum string length and the NULL-terminator requirement.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_fapl_info_t {
uint8_t version;
hid_t backing_fapl_id;
uint32_t page_size;
H5FD_onion_target_file_constant_t store_target;
uint64_t revision_num;
uint8_t force_write_open;
uint8_t creation_flags;
char comment[H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN + 1];
} H5FD_onion_fapl_info_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL hid_t H5FD_onion_init(void);
/**
* --------------------------------------------------------------------------
* \ingroup H5P
*
* \brief get the onion info from the file access property list
*
* \param[in] fapl_id The ID of the file access property list
* \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t
*
* \return \herr_t
*
* \details H5Pget_fapl_onion() retrieves the structure H5FD_onion_fapl_info_t
* from the file access property list that is set for the onion VFD
* driver.
*/
H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out);
/**
* --------------------------------------------------------------------------
* \ingroup H5P
*
* \brief set the onion info for the file access property list
*
* \param[in] fapl_id The ID of the file access property list
* \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t
*
* \return \herr_t
*
* \details H5Pset_fapl_onion() sets the structure H5FD_onion_fapl_info_t
* for the file access property list that is set for the onion VFD
* driver.
*/
H5_DLL herr_t H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa);
/**
* --------------------------------------------------------------------------
* \ingroup H5FD
*
* \brief get the number of revisions
*
* \param[in] filename The name of the onion file
* \param[in] fapl_id The ID of the file access property list
* \param[out] revision_count The number of revisions
*
* \return \herr_t
*
* \details H5FDonion_get_revision_count() returns the number of revisions
* for an onion file. It takes the file name and file access property
* list that is set for the onion VFD driver.
*
*/
H5_DLL herr_t H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_H */

231
src/H5FDonion_header.c Normal file
View File

@ -0,0 +1,231 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the onion file's header
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/*-----------------------------------------------------------------------------
* Function: H5FD_ingest_header
*
* Purpose: Read and decode the history header information from `raw_file`
* at `addr`, and store the decoded information in the structure
* at `hdr_out`.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr)
{
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
haddr_t size = (haddr_t)H5FD_ONION_ENCODED_SIZE_HEADER;
uint32_t sum = 0;
FUNC_ENTER_PACKAGE;
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF")
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history header from file")
if (H5FD__onion_header_decode(buf, hdr_out) == 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history header")
sum = H5_checksum_fletcher32(buf, size - 4);
if (hdr_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_header() */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_write_header
*
* Purpose: Write in-memory history header to appropriate backing file.
* Overwrites existing header data.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file)
{
uint32_t sum = 0; /* Not used, but required by the encoder */
uint64_t size = 0;
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer for updated history header")
if (0 == (size = H5FD__onion_header_encode(header, buf, &sum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding updated history header")
if (H5FD_write(file, H5FD_MEM_DRAW, 0, (haddr_t)size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header")
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write_header()*/
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_header_decode
*
* Purpose: Attempt to read a buffer and store it as a history-header
* structure.
*
* Implementation must correspond with
* H5FD__onion_header_encode().
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header)
{
uint32_t ui32 = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(header != NULL);
HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version);
if (HDstrncmp((const char *)buf, H5FD_ONION_HEADER_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header signature")
if (buf[4] != H5FD_ONION_HEADER_VERSION_CURR)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header version")
ptr = buf + 5;
ui32 = 0;
HDmemcpy(&ui32, ptr, 3);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->flags);
ptr += 3;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->page_size);
ptr += 4;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->origin_eof);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->history_addr);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->history_size);
ptr += 8;
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->checksum);
ptr += 4;
if (sum != header->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_header_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_header_encode
*
* Purpose: Write history-header structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_header_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents (H5FD_ONION_ENCODED_SIZE_HEADER).
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum /*out*/)
{
unsigned char *ptr = buf;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(buf != NULL);
HDassert(checksum != NULL);
HDassert(header != NULL);
HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version);
HDassert(0 == (header->flags & 0xFF000000)); /* max three bits long */
HDmemcpy(ptr, H5FD_ONION_HEADER_SIGNATURE, 4);
ptr += 4;
HDmemcpy(ptr, (unsigned char *)&header->version, 1);
ptr += 1;
UINT32ENCODE(ptr, header->flags);
ptr -= 1; /* truncate to three bytes */
UINT32ENCODE(ptr, header->page_size);
UINT64ENCODE(ptr, header->origin_eof);
UINT64ENCODE(ptr, header->history_addr);
UINT64ENCODE(ptr, header->history_size);
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
ret_value = (size_t)(ptr - buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_header_encode() */

56
src/H5FDonion_header.h Normal file
View File

@ -0,0 +1,56 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the onion file's header
*/
#ifndef H5FDonion_header_H
#define H5FDonion_header_H
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_HEADER 40
/* Flags must align exactly one per bit, up to 24 bits */
#define H5FD_ONION_HEADER_FLAG_WRITE_LOCK 0x1
#define H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY 0x2
#define H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT 0x4
#define H5FD_ONION_HEADER_SIGNATURE "OHDH"
#define H5FD_ONION_HEADER_VERSION_CURR 1
/* In-memory representation of the on-store onion history file header.
*/
typedef struct H5FD_onion_header_t {
uint8_t version;
uint32_t flags; /* At most three bytes used! */
uint32_t page_size;
uint64_t origin_eof; /* Size of the 'original' canonical file */
uint64_t history_addr;
uint64_t history_size;
uint32_t checksum;
} H5FD_onion_header_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr);
H5_DLL herr_t H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file);
H5_DLL size_t H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header);
H5_DLL size_t H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_header_H */

305
src/H5FDonion_history.c Normal file
View File

@ -0,0 +1,305 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the onion file's history
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write_history
*
* Purpose: Read and decode the history information from `raw_file` at
* `addr` .. `addr + size` (taken from history header), and store
* the decoded information in the structure at `history_out`.
*
* Returns: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr, haddr_t size)
{
unsigned char *buf = NULL;
uint32_t sum = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(history_out);
HDassert(raw_file);
/* Set early so we can clean up properly on errors */
history_out->record_locs = NULL;
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF");
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer space");
if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA");
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history from file");
if (H5FD__onion_history_decode(buf, history_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (initial)");
sum = H5_checksum_fletcher32(buf, size - 4);
if (history_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored");
if (history_out->n_revisions > 0)
if (NULL == (history_out->record_locs =
H5MM_calloc(history_out->n_revisions * sizeof(H5FD_onion_record_loc_t))))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate record pointer list");
if (H5FD__onion_history_decode(buf, history_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (final)");
done:
H5MM_xfree(buf);
if (ret_value < 0)
H5MM_xfree(history_out->record_locs);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_history() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write_history
*
* Purpose: Encode and write history to file at the given address.
*
* Returns: Success: Number of bytes written to destination file (always non-zero)
* Failure: 0
*-----------------------------------------------------------------------------
*/
uint64_t
H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start,
haddr_t filesize_curr)
{
uint32_t _sum = 0; /* Required by the API call but unused here */
uint64_t size = 0;
unsigned char *buf = NULL;
uint64_t ret_value = 0;
FUNC_ENTER_PACKAGE;
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY +
(H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * history->n_revisions))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, "can't allocate buffer for updated history")
if (0 == (size = H5FD__onion_history_encode(history, buf, &_sum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, 0, "problem encoding updated history")
if ((size + off_start > filesize_curr) && (H5FD_set_eoa(file, H5FD_MEM_DRAW, off_start + size) < 0))
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, 0, "can't modify EOA for updated history")
if (H5FD_write(file, H5FD_MEM_DRAW, off_start, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, 0, "can't write history as intended")
ret_value = size;
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write_history() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_history_decode
*
* Purpose: Attempt to read a buffer and store it as a history
* structure.
*
* Implementation must correspond with
* H5FD__onion_history_encode().
*
* MUST BE CALLED TWICE:
* On the first call, n_records in the destination structure must
* be zero, and record_locs be NULL.
*
* If the buffer is well-formed, the destination structure is
* tentatively populated with fixed-size values, and the number of
* bytes read are returned.
*
* Prior to the second call, the user must allocate space for
* record_locs to hold n_records record-pointer structs.
*
* Then the decode operation is called a second time, and all
* components will be populated (and again number of bytes read is
* returned).
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history)
{
uint32_t ui32 = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint64_t n_revisions = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(history != NULL);
HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version);
if (HDstrncmp((const char *)buf, H5FD_ONION_HISTORY_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature")
if (H5FD_ONION_HISTORY_VERSION_CURR != buf[4])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid version")
ptr = buf + 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, n_revisions);
ptr += 8;
if (0 == history->n_revisions) {
history->n_revisions = n_revisions;
ptr += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * n_revisions;
}
else {
if (history->n_revisions != n_revisions)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
"history argument suggests different revision count than encoded buffer")
if (NULL == history->record_locs)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "list is NULL -- cannot populate")
for (uint64_t i = 0; i < n_revisions; i++) {
H5FD_onion_record_loc_t *rloc = &history->record_locs[i];
/* Decode into appropriately sized types, then do a checked
* assignment to the struct value. We don't have access to
* the H5F_t struct for this file, so we can't use the
* offset/length macros in H5Fprivate.h.
*/
uint64_t record_size;
uint64_t phys_addr;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, phys_addr);
H5_CHECKED_ASSIGN(rloc->phys_addr, haddr_t, phys_addr, uint64_t);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record_size);
H5_CHECKED_ASSIGN(rloc->record_size, hsize_t, record_size, uint64_t);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, rloc->checksum);
ptr += 4;
}
}
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, history->checksum);
ptr += 4;
if (sum != history->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_history_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_history_encode
*
* Purpose: Write history structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_history_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents.
* (Hint: `sizeof(history struct) +
* sizeof(record-pointer-struct) * n_records)` guarantees
* ample/excess space.)
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf, uint32_t *checksum)
{
unsigned char *ptr = buf;
size_t vers_u32 = (uint32_t)history->version; /* pad out unused bytes */
FUNC_ENTER_PACKAGE_NOERR;
HDassert(history != NULL);
HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version);
HDassert(buf != NULL);
HDassert(checksum != NULL);
HDmemcpy(ptr, H5FD_ONION_HISTORY_SIGNATURE, 4);
ptr += 4;
UINT32ENCODE(ptr, vers_u32);
UINT64ENCODE(ptr, history->n_revisions);
if (history->n_revisions > 0) {
HDassert(history->record_locs != NULL);
for (uint64_t i = 0; i < history->n_revisions; i++) {
H5FD_onion_record_loc_t *rloc = &history->record_locs[i];
/* Do a checked assignment from the struct value into appropriately
* sized types. We don't have access to the H5F_t struct for this
* file, so we can't use the offset/length macros in H5Fprivate.h.
*/
uint64_t phys_addr;
uint64_t record_size;
H5_CHECKED_ASSIGN(phys_addr, uint64_t, rloc->phys_addr, haddr_t);
H5_CHECKED_ASSIGN(record_size, uint64_t, rloc->record_size, hsize_t);
UINT64ENCODE(ptr, phys_addr);
UINT64ENCODE(ptr, record_size);
UINT32ENCODE(ptr, rloc->checksum);
}
}
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
FUNC_LEAVE_NOAPI((size_t)(ptr - buf));
} /* end H5FD__onion_history_encode() */

63
src/H5FDonion_history.h Normal file
View File

@ -0,0 +1,63 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the onion file's history
*/
#ifndef H5FDonion_history_H
#define H5FDonion_history_H
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_HISTORY 20
#define H5FD_ONION_HISTORY_SIGNATURE "OWHS"
#define H5FD_ONION_HISTORY_VERSION_CURR 1
/* In-memory representation of the on-store revision record.
* Used in the history.
*/
typedef struct H5FD_onion_record_loc_t {
haddr_t phys_addr;
hsize_t record_size;
uint32_t checksum;
} H5FD_onion_record_loc_t;
/* In-memory representation of the on-store history record/summary.
*/
typedef struct H5FD_onion_history_t {
uint8_t version;
uint64_t n_revisions;
H5FD_onion_record_loc_t *record_locs;
uint32_t checksum;
} H5FD_onion_history_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr,
haddr_t size);
H5_DLL uint64_t H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start,
haddr_t filesize_curr);
H5_DLL size_t H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history);
H5_DLL size_t H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf,
uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_history_H */

935
src/H5FDonion_index.c Normal file
View File

@ -0,0 +1,935 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the archival and revision indexes
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */
#define U64_EXP2(n) ((uint64_t)1 << (n))
static int H5FD__onion_archival_index_list_sort_cmp(const void *, const void *);
static herr_t H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix);
/*-----------------------------------------------------------------------------
* Read and decode the revision_record information from `raw_file` at
* `addr` .. `addr + size` (taken from history), and store the decoded
* information in the structure at `r_out`.
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file,
const H5FD_onion_history_t *history, uint64_t revision_num)
{
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
uint64_t n = 0;
uint64_t high = 0;
uint64_t low = 0;
uint64_t range = 0;
uint32_t sum = 0;
haddr_t addr = 0;
size_t size = 0;
FUNC_ENTER_PACKAGE;
HDassert(r_out);
HDassert(raw_file);
HDassert(history);
HDassert(history->record_locs);
HDassert(history->n_revisions > 0);
high = history->n_revisions - 1;
range = high;
addr = history->record_locs[high].phys_addr;
size = history->record_locs[high].record_size;
/* Initialize r_out
*
* TODO: This function should completely initialize r_out. Relying on
* other code to some of the work while we just paste over parts
* of the struct here is completely bananas.
*/
r_out->comment = H5MM_xfree(r_out->comment);
r_out->archival_index.list = H5MM_xfree(r_out->archival_index.list);
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "at least one record extends beyond EOF")
/* recovery-open may have EOA below revision record */
if ((H5FD_get_eoa(raw_file, H5FD_MEM_DRAW) < (addr + size)) &&
(H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)) {
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA");
}
/* Perform binary search on records to find target revision by ID.
* As IDs are added sequentially, they are guaranteed to be sorted.
*/
while (range > 0) {
n = (range / 2) + low;
addr = history->record_locs[n].phys_addr;
size = history->record_locs[n].record_size;
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)")
sum = H5_checksum_fletcher32(buf, size - 4);
if (r_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
if (revision_num == r_out->revision_num)
break;
H5MM_xfree(buf);
buf = NULL;
r_out->archival_index.n_entries = 0;
r_out->comment_size = 0;
if (r_out->revision_num < revision_num)
low = (n == high) ? high : n + 1;
else
high = (n == low) ? low : n - 1;
range = high - low;
} /* end while 'non-leaf' binary search */
if (range == 0) {
n = low;
addr = history->record_locs[n].phys_addr;
size = history->record_locs[n].record_size;
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)")
sum = H5_checksum_fletcher32(buf, size - 4);
if (r_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
if (revision_num != r_out->revision_num)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL,
"could not find target revision!") /* TODO: corrupted? */
} /* end if revision ID at 'leaf' in binary search */
if (r_out->comment_size > 0)
if (NULL == (r_out->comment = H5MM_malloc(sizeof(char) * r_out->comment_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate comment space")
if (r_out->archival_index.n_entries > 0)
if (NULL == (r_out->archival_index.list =
H5MM_calloc(r_out->archival_index.n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate index entry list")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (final)")
done:
H5MM_xfree(buf);
if (ret_value == FAIL) {
H5MM_xfree(r_out->comment);
H5MM_xfree(r_out->archival_index.list);
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_revision_record() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_archival_index_is_valid
*
* Purpose: Determine whether an archival index structure is valid.
*
* + Verify page size (power of two).
* + Verify list exists.
* + Verify list contents:
* + Sorted by increasing logical address (no duplicates)
* + Logical addresses are multiples of page size.
*
* Return: TRUE/FALSE
*-----------------------------------------------------------------------------
*/
hbool_t
H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *aix)
{
hbool_t ret_value = TRUE;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(aix);
if (H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR != aix->version)
HGOTO_DONE(FALSE)
if (NULL == aix->list)
HGOTO_DONE(FALSE)
/* Ensure list is sorted on logical_page field */
if (aix->n_entries > 1)
for (uint64_t i = 1; i < aix->n_entries - 1; i++)
if (aix->list[i + 1].logical_page <= aix->list[i].logical_page)
HGOTO_DONE(FALSE)
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_archival_index_is_valid() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_archival_index_find
*
* Purpose: Retrieve the archival index entry by logical page ID.
*
* The archival index pointer must point to a valid index entry.
* The entry out pointer-pointer cannot be null.
*
* Return: Success: Positive value (1) -- entry found.
* Entry out pointer-pointer is set to point to entry.
* Failure: Zero (0) -- entry not found.
* Entry out pointer-pointer is unmodified.
*-----------------------------------------------------------------------------
*/
int
H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *aix, uint64_t logical_page,
const H5FD_onion_index_entry_t **entry_out)
{
uint64_t low = 0;
uint64_t high = 0;
uint64_t n = 0;
uint64_t range = 0;
H5FD_onion_index_entry_t *x = NULL;
int ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(aix);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version);
HDassert(entry_out);
if (aix->n_entries != 0)
HDassert(aix->list);
high = aix->n_entries - 1;
range = high;
/* Trivial cases */
if (aix->n_entries == 0 || logical_page > aix->list[high].logical_page ||
logical_page < aix->list[0].logical_page)
HGOTO_DONE(0)
/*
* Binary search on sorted list
*/
/* Winnow down to first of found or one element */
while (range > 0) {
HDassert(high < aix->n_entries);
n = low + (range / 2);
x = &(aix->list[n]);
if (x->logical_page == logical_page) {
*entry_out = x; /* element found at fence */
ret_value = 1;
goto done;
}
else if (x->logical_page < logical_page) {
low = (n == high) ? high : n + 1;
}
else {
high = (n == low) ? low : n - 1;
}
range = high - low;
}
HDassert(high == low); /* one element */
/* n == low/high check because we may have tested it already above */
if ((n != low || n != high) && (aix->list[low].logical_page == logical_page)) {
*entry_out = &aix->list[low];
ret_value = 1;
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_archival_index_find() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_destroy
*
* Purpose: Release all resources of a revision index.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *rix)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
for (size_t i = 0; 0 < rix->_hash_table_n_keys_populated && i < rix->_hash_table_size; i++) {
H5FD_onion_revision_index_hash_chain_node_t *next = NULL;
H5FD_onion_revision_index_hash_chain_node_t *node = rix->_hash_table[i];
if (node != NULL)
rix->_hash_table_n_keys_populated -= 1;
while (node != NULL) {
HDassert(H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR == node->version);
next = node->next;
H5MM_xfree(node);
node = next;
}
}
H5MM_xfree(rix->_hash_table);
H5MM_xfree(rix);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_destroy() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_init
*
* Purpose: Initialize a revision index structure with a default starting
* size. A new structure is allocated and populated with initial
* values.
*
* Return: Success: Pointer to newly-allocated structure
* Failure: NULL
*-----------------------------------------------------------------------------
*/
H5FD_onion_revision_index_t *
H5FD__onion_revision_index_init(uint32_t page_size)
{
uint64_t table_size = U64_EXP2(H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2);
H5FD_onion_revision_index_t *rix = NULL;
H5FD_onion_revision_index_t *ret_value = NULL;
FUNC_ENTER_PACKAGE;
HDassert(0 != page_size);
HDassert(POWER_OF_TWO(page_size));
if (NULL == (rix = H5MM_calloc(sizeof(H5FD_onion_revision_index_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate index")
if (NULL ==
(rix->_hash_table = H5MM_calloc(table_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate hash table")
rix->version = H5FD_ONION_REVISION_INDEX_VERSION_CURR;
rix->n_entries = 0;
/* Compute and store log2(page_size) */
for (rix->page_size_log2 = 0; (((uint32_t)1 << rix->page_size_log2) & page_size) == 0;
rix->page_size_log2++)
;
rix->_hash_table_size = table_size;
rix->_hash_table_size_log2 = H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2;
rix->_hash_table_n_keys_populated = 0;
ret_value = rix;
done:
if (NULL == ret_value)
H5MM_xfree(rix);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_init() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_resize()
*
* Purpose: Replace the hash table in the revision index.
*
* Doubles the available number of keys, re-hashes table contents,
* and updates relevant components in the index structure.
*
* Fails if unable to allocate space for larger hash table.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix)
{
H5FD_onion_revision_index_hash_chain_node_t **new_table = NULL;
uint64_t new_size_log2 = rix->_hash_table_size_log2 + 1;
uint64_t new_size = U64_EXP2(new_size_log2);
uint64_t new_n_keys_populated = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(rix->_hash_table);
if (NULL == (new_table = H5MM_calloc(new_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new hash table")
for (uint64_t i = 0; i < rix->_hash_table_size; i++) {
while (rix->_hash_table[i] != NULL) {
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
uint64_t key = 0;
/* Pop entry off of bucket stack and re-hash */
node = rix->_hash_table[i];
rix->_hash_table[i] = node->next;
node->next = NULL;
key = node->entry_data.logical_page & (new_size - 1);
if (NULL == new_table[key]) {
new_table[key] = node;
new_n_keys_populated++;
}
else {
node->next = new_table[i];
new_table[i] = node;
}
}
}
H5MM_xfree(rix->_hash_table);
rix->_hash_table_size = new_size;
rix->_hash_table_size_log2 = new_size_log2;
rix->_hash_table_n_keys_populated = new_n_keys_populated;
rix->_hash_table = new_table;
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_resize() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_insert()
*
* Purpose: Add an entry to the revision index, or update an existing
* entry. Must be used to update entries as well as add --
* checksum value will change.
*
* Entry data is copied into separate memory region; user pointer
* can be safley re-used or discarded after operation.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *rix, const H5FD_onion_index_entry_t *entry)
{
uint64_t key = 0;
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
H5FD_onion_revision_index_hash_chain_node_t **append_dest = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(entry);
/* Resize and re-hash table if necessary */
if (rix->n_entries >= (rix->_hash_table_size * 2) ||
rix->_hash_table_n_keys_populated >= (rix->_hash_table_size / 2)) {
if (H5FD__onion_revision_index_resize(rix) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NONE_MINOR, FAIL, "unable to resize and hash table")
}
key = entry->logical_page & (rix->_hash_table_size - 1);
HDassert(key < rix->_hash_table_size);
if (NULL == rix->_hash_table[key]) {
/* Key maps to empty bucket */
append_dest = &rix->_hash_table[key];
rix->_hash_table_n_keys_populated++;
}
else {
/* Key maps to populated bucket */
for (node = rix->_hash_table[key]; node != NULL; node = node->next) {
append_dest = &node->next; /* look for bucket tail */
if (entry->logical_page == node->entry_data.logical_page) {
if (entry->phys_addr != node->entry_data.phys_addr) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "physical address mismatch");
}
HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t));
append_dest = NULL; /* Node updated, do not append */
break;
}
}
}
/* Add new entry to bucket chain */
if (append_dest != NULL) {
if (NULL == (node = H5MM_malloc(sizeof(H5FD_onion_revision_index_hash_chain_node_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new ash chain node")
node->version = H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR;
node->next = NULL;
HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t));
*append_dest = node;
rix->n_entries++;
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_insert() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_find()
*
*
* Purpose: Get pointer to revision index entry with the given page number,
* if it exists in the index.
*
* Return: Success: Positive value (1) -- entry found.
* Entry out pointer-pointer is set to point to entry.
* Failure: Zero (0) -- entry not found.
* Entry out pointer-pointer is unmodified.
*-----------------------------------------------------------------------------
*/
int
H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *rix, uint64_t logical_page,
const H5FD_onion_index_entry_t **entry_out)
{
uint64_t key = 0;
int ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(rix->_hash_table);
HDassert(entry_out);
key = logical_page & (rix->_hash_table_size - 1);
HDassert(key < rix->_hash_table_size);
if (rix->_hash_table[key] != NULL) {
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
for (node = rix->_hash_table[key]; node != NULL; node = node->next) {
if (logical_page == node->entry_data.logical_page) {
*entry_out = &node->entry_data;
ret_value = 1;
break;
}
}
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_find() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_record_decode
*
* Purpose: Attempt to read a buffer and store it as a revision record
* structure.
*
* Implementation must correspond with
* H5FD__onion_revision_record_encode().
*
* MUST BE CALLED TWICE:
* On the first call, n_entries and comment_size in the
* destination structure must all all be zero, and their
* respective variable-length components (index_entry_list,
* comment) must all be NULL.
*
* If the buffer is well-formed, the destination structure is
* tentatively populated with fixed-size values, and the number of
* bytes read are returned.
*
* Prior to the second call, the user must allocate space for the
* variable-length components, in accordance with the associated
* indicators (array of index-entry structures for
* index_entry_list, of size n_entries; character arrays for
* comment, allocated with the *_size number of bytes -- space
* for NULL-terminator is included in _size).
*
* Then the decode operation is called a second time, and all
* components will be populated (and again number of bytes read is
* returned).
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record)
{
uint32_t ui32 = 0;
uint32_t page_size = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint64_t n_entries = 0;
uint32_t comment_size = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(record != NULL);
HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version);
if (HDstrncmp((const char *)buf, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature")
if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != buf[4])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid record version")
ptr = buf + 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->revision_num);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->parent_revision_num);
ptr += 8;
HDmemcpy(record->time_of_creation, ptr, 16);
ptr += 16;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->logical_eof);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, page_size);
ptr += 4;
if (page_size == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size is zero")
if (!POWER_OF_TWO(page_size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size not power of two")
for (record->archival_index.page_size_log2 = 0;
(((uint32_t)1 << record->archival_index.page_size_log2) & page_size) == 0;
record->archival_index.page_size_log2++)
;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, n_entries);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, comment_size);
ptr += 4;
if (record->archival_index.n_entries == 0) {
record->archival_index.n_entries = n_entries;
ptr += H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * n_entries;
}
else if (n_entries != record->archival_index.n_entries) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "n_entries in archival index does not match decoded")
}
else {
H5FD_onion_index_entry_t *entry = NULL;
if (record->archival_index.list == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no archival index entry list")
for (size_t i = 0; i < n_entries; i++) {
entry = &record->archival_index.list[i];
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, entry->logical_page);
ptr += 8;
/* logical_page actually encoded as address; check and convert */
if (entry->logical_page & (page_size - 1))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "logical address does not align with page size")
entry->logical_page = entry->logical_page >> record->archival_index.page_size_log2;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, entry->phys_addr);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, sum);
ptr += 4;
ui32 = H5_checksum_fletcher32((ptr - 20), 16);
if (ui32 != sum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "index entry checksum mismatch")
}
}
if (record->comment_size == 0) {
if (record->comment != NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "comment pointer prematurely allocated")
record->comment_size = comment_size;
}
else {
if (record->comment == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no comment pointer")
HDmemcpy(record->comment, ptr, comment_size);
}
ptr += comment_size;
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, record->checksum);
ptr += 4;
if (sum != record->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_record_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_record_encode
*
* Purpose: Write revision-record structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_revision_record_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents.
* (Hint: `sizeof(revision-record-struct) + comment-size +
* sizeof(index-entry-struct) * n_entries)`
* guarantees ample/excess space.)
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf,
uint32_t *checksum)
{
unsigned char *ptr = buf; /* original pointer */
uint32_t vers_u32 = (uint32_t)record->version; /* pad out unused bytes */
uint32_t page_size = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(checksum != NULL);
HDassert(buf != NULL);
HDassert(record != NULL);
HDassert(vers_u32 < 0x100);
HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version);
page_size = (uint32_t)(1 << record->archival_index.page_size_log2);
HDmemcpy(ptr, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4);
ptr += 4;
UINT32ENCODE(ptr, vers_u32);
UINT64ENCODE(ptr, record->revision_num);
UINT64ENCODE(ptr, record->parent_revision_num);
HDmemcpy(ptr, record->time_of_creation, 16);
ptr += 16;
UINT64ENCODE(ptr, record->logical_eof);
UINT32ENCODE(ptr, page_size);
UINT64ENCODE(ptr, record->archival_index.n_entries);
UINT32ENCODE(ptr, record->comment_size);
if (record->archival_index.n_entries > 0) {
uint64_t page_size_log2 = record->archival_index.page_size_log2;
HDassert(record->archival_index.list != NULL);
for (uint64_t i = 0; i < record->archival_index.n_entries; i++) {
uint32_t sum = 0;
H5FD_onion_index_entry_t *entry = NULL;
uint64_t logi_addr = 0;
entry = &record->archival_index.list[i];
logi_addr = entry->logical_page << page_size_log2;
UINT64ENCODE(ptr, logi_addr);
UINT64ENCODE(ptr, entry->phys_addr);
sum = H5_checksum_fletcher32((ptr - 16), 16);
UINT32ENCODE(ptr, sum);
}
}
if (record->comment_size > 0) {
HDassert(record->comment != NULL && *record->comment != '\0');
HDmemcpy(ptr, record->comment, record->comment_size);
ptr += record->comment_size;
}
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
FUNC_LEAVE_NOAPI((size_t)(ptr - buf));
} /* end H5FD__onion_revision_record_encode() */
/*-----------------------------------------------------------------------------
* Callback for comparisons in sorting archival index entries by logical_page.
*-----------------------------------------------------------------------------
*/
static int
H5FD__onion_archival_index_list_sort_cmp(const void *_a, const void *_b)
{
const H5FD_onion_index_entry_t *a = (const H5FD_onion_index_entry_t *)_a;
const H5FD_onion_index_entry_t *b = (const H5FD_onion_index_entry_t *)_b;
if (a->logical_page < b->logical_page)
return -1;
else if (a->logical_page > b->logical_page)
return 1;
return 0;
} /* end H5FD__onion_archival_index_list_sort_cmp() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_merge_revision_index_into_archival_index
*
* Purpose: Merge index entries from revision index into archival index.
*
* If successful, the archival index is expanded 'behind the
* scenes' and new entries from the revision index are inserted.
* The archival index remains sorted in ascending order of logical
* address.
*
* The conversion to archival index changes logical pages in
* revision index entries to their logical addresses in-file.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *rix,
H5FD_onion_archival_index_t *aix)
{
uint64_t n_kept = 0;
H5FD_onion_index_entry_t *kept_list = NULL;
H5FD_onion_archival_index_t new_aix = {
H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, 0, /* page_size_log2 tbd */
0, /* n_entries */
NULL, /* list pointer (allocated later) */
};
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(aix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version);
HDassert(aix->page_size_log2 == rix->page_size_log2);
/* If the revision index is empty there is nothing to archive */
if (rix->n_entries == 0)
goto done;
/* Add all revision index entries to new archival list */
new_aix.page_size_log2 = aix->page_size_log2;
if (NULL == (new_aix.list = H5MM_calloc(rix->n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate new archival index list")
for (uint64_t i = 0; i < rix->_hash_table_size; i++) {
const H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
for (node = rix->_hash_table[i]; node != NULL; node = node->next) {
HDmemcpy(&new_aix.list[new_aix.n_entries], &node->entry_data, sizeof(H5FD_onion_index_entry_t));
new_aix.n_entries++;
}
}
/* Sort the new archival list */
HDqsort(new_aix.list, new_aix.n_entries, sizeof(H5FD_onion_index_entry_t),
H5FD__onion_archival_index_list_sort_cmp);
/* Add the old archival index entries to a 'kept' list containing the
* old archival list entries that are not also included in the revision
* list.
*
* Note that kept_list will be NULL if there are no entries in the passed-in
* archival list.
*/
if (aix->n_entries > 0)
if (NULL == (kept_list = H5MM_calloc(aix->n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate larger archival index list")
for (uint64_t i = 0; i < aix->n_entries; i++) {
const H5FD_onion_index_entry_t *entry = NULL;
/* Add only if page not already added from revision index */
if (H5FD__onion_archival_index_find(&new_aix, aix->list[i].logical_page, &entry) == 0) {
HDmemcpy(&kept_list[n_kept], &aix->list[i], sizeof(H5FD_onion_index_entry_t));
n_kept++;
}
}
/* Destroy the old archival list and replace with a list big enough to hold
* the revision list entries and the kept list entries
*/
H5MM_xfree(aix->list);
if (NULL == (aix->list = H5MM_calloc((new_aix.n_entries + n_kept) * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate exact-size archival index list")
/* Copy (new) revision list entries to replacement list */
HDmemcpy(aix->list, new_aix.list, sizeof(H5FD_onion_index_entry_t) * new_aix.n_entries);
aix->n_entries = new_aix.n_entries;
/* Copy (old) kept archival list entries to replacement list */
if (n_kept > 0) {
HDmemcpy(&aix->list[aix->n_entries], kept_list, sizeof(H5FD_onion_index_entry_t) * n_kept);
aix->n_entries += n_kept;
}
/* Sort this list */
HDqsort(aix->list, aix->n_entries, sizeof(H5FD_onion_index_entry_t),
H5FD__onion_archival_index_list_sort_cmp);
done:
/* Free the temporary lists */
H5MM_xfree(kept_list);
H5MM_xfree(new_aix.list);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_merge_revision_index_into_archival_index() */

150
src/H5FDonion_index.h Normal file
View File

@ -0,0 +1,150 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef H5FDonion_index_H
#define H5FDonion_index_H
#define H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR 1
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY 20
#define H5FD_ONION_ENCODED_SIZE_RECORD_POINTER 20
#define H5FD_ONION_ENCODED_SIZE_REVISION_RECORD 68
#define H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR 1
#define H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2 10 /* 2^n slots */
#define H5FD_ONION_REVISION_INDEX_VERSION_CURR 1
#define H5FD_ONION_REVISION_RECORD_SIGNATURE "ORRS"
#define H5FD_ONION_REVISION_RECORD_VERSION_CURR 1
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the archival and revision indexes
*/
/*-----------------------------------------------------------------------------
*
* Structure H5FD__onion_index_entry
*
* Purpose: Map a page in the logical file to a 'physical address' in the
* onion file.
*
* logical_page:
*
* Page 'id' in the logical file.
*
* phys_addr:
*
* Address/offset of start of page in the onion file.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_index_entry_t {
uint64_t logical_page;
haddr_t phys_addr;
} H5FD_onion_index_entry_t;
/*-----------------------------------------------------------------------------
*
* Structure H5FD__onion_archival_index
*
* Purpose: Encapsulate archival index and associated data.
* Convenience structure with sanity-checking components.
*
* version: Future-proofing identifier. Informs struct membership.
* Must equal H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR to be
* considered valid.
*
* page_size: Interval to which the `logical_page` component of each list
* entry must align.
* Value is taken from the onion history data; must not change
* following onionization or file or creation of onion file.
*
* n_entries: Number of entries in the list.
*
* list: Pointer to array of archival index entries.
* Cannot be NULL.
* Entries must be sorted by `logical_page_id` in ascending order.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_archival_index_t {
uint8_t version;
uint32_t page_size_log2;
uint64_t n_entries;
H5FD_onion_index_entry_t *list;
} H5FD_onion_archival_index_t;
/* data structure for storing index entries at a hash key collision */
/* version 1 implements a singly-linked list */
typedef struct H5FD_onion_revision_index_hash_chain_node_t H5FD_onion_revision_index_hash_chain_node_t;
struct H5FD_onion_revision_index_hash_chain_node_t {
uint8_t version;
H5FD_onion_index_entry_t entry_data;
H5FD_onion_revision_index_hash_chain_node_t *next;
};
typedef struct H5FD_onion_revision_index_t {
uint8_t version;
uint32_t page_size_log2;
uint64_t n_entries; /* count of all entries in table */
uint64_t _hash_table_size; /* 'slots' in hash table */
uint64_t _hash_table_size_log2; /* 2^(n) -> 'slots' in hash table */
uint64_t _hash_table_n_keys_populated; /* count of slots not NULL */
H5FD_onion_revision_index_hash_chain_node_t **_hash_table;
} H5FD_onion_revision_index_t;
/* In-memory representation of the on-store revision record.
*/
typedef struct H5FD_onion_revision_record_t {
uint8_t version;
uint64_t revision_num;
uint64_t parent_revision_num;
char time_of_creation[16];
uint64_t logical_eof;
H5FD_onion_archival_index_t archival_index;
uint32_t comment_size;
char *comment;
uint32_t checksum;
} H5FD_onion_revision_record_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file,
const H5FD_onion_history_t *history, uint64_t revision_num);
H5_DLL hbool_t H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *);
H5_DLL int H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *, uint64_t,
const H5FD_onion_index_entry_t **);
H5_DLL H5FD_onion_revision_index_t *H5FD__onion_revision_index_init(uint32_t page_size);
H5_DLL herr_t H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *);
H5_DLL herr_t H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *,
const H5FD_onion_index_entry_t *);
H5_DLL int H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *, uint64_t,
const H5FD_onion_index_entry_t **);
H5_DLL herr_t H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *,
H5FD_onion_archival_index_t *);
H5_DLL size_t H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record);
H5_DLL size_t H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf,
uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_index_H */

28
src/H5FDonion_priv.h Normal file
View File

@ -0,0 +1,28 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD) Internals.
*
* Purpose: The private header file for the Onion VFD.
* Contains definitions and declarations used internallay and by
* tests.
*/
#ifndef H5FDonion_priv_H
#define H5FDonion_priv_H
#include "H5FDonion_header.h"
#include "H5FDonion_history.h"
#include "H5FDonion_index.h"
#endif /* H5FDonion_priv_H */

1
src/H5FDpublic.h
View File

@ -46,6 +46,7 @@
#define H5_VFD_ROS3 ((H5FD_class_value_t)(11))
#define H5_VFD_SUBFILING ((H5FD_class_value_t)(12))
#define H5_VFD_IOC ((H5FD_class_value_t)(13))
#define H5_VFD_ONION ((H5FD_class_value_t)(14))
/* VFD IDs below this value are reserved for library use. */
#define H5_VFD_RESERVED 256

3
src/H5private.h
View File

@ -1068,6 +1068,9 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation);
#ifndef HDlog
#define HDlog(X) log(X)
#endif
#ifndef HDlog2
#define HDlog2(X) log2(X)
#endif
#ifndef HDlog10
#define HDlog10(X) log10(X)
#endif

3
src/H5trace.c
View File

@ -1098,6 +1098,9 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap)
H5RS_acat(rs, "H5_VFD_ROS3");
break;
#endif
case H5_VFD_ONION:
H5RS_acat(rs, "H5_VFD_ONION");
break;
default:
H5RS_asprintf_cat(rs, "%ld", (long)class_val);
break;

12
src/Makefile.am
View File

@ -61,8 +61,9 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \
H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
H5FAint.c H5FAstat.c H5FAtest.c \
H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c \
H5FDmulti.c H5FDperform.c H5FDsec2.c H5FDspace.c \
H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmulti.c \
H5FDonion.c H5FDonion_header.c H5FDonion_history.c H5FDonion_index.c \
H5FDperform.c H5FDsec2.c H5FDspace.c \
H5FDsplitter.c H5FDstdio.c H5FDtest.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \
H5FSstat.c H5FStest.c \
@ -154,9 +155,10 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
H5Cpublic.h H5Dpublic.h \
H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \
H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \
H5FDsec2.h H5FDsplitter.h \
H5FDstdio.h H5FDwindows.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \
H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h \
H5FDonion.h H5FDros3.h H5FDsec2.h H5FDsplitter.h \
H5FDstdio.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \
H5FDwindows.h \
H5Gpublic.h H5Ipublic.h H5Lpublic.h \
H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \
H5PLextern.h H5PLpublic.h \

1
src/hdf5.h
View File

@ -64,6 +64,7 @@
#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */
#include "H5FDmpi.h" /* MPI-based file drivers */
#include "H5FDmulti.h" /* Usage-partitioned file family */
#include "H5FDonion.h" /* Onion file I/O */
#include "H5FDros3.h" /* R/O S3 "file" I/O */
#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */

1
test/CMakeLists.txt
View File

@ -381,6 +381,7 @@ set (H5_TESTS
timer
cmpd_dtransform
event_set
onion
)
if (HDF5_BUILD_UTILS)
set (H5_TESTS ${H5_TESTS} mirror_vfd)

3
test/Makefile.am
View File

@ -70,7 +70,8 @@ TEST_PROG= testhdf5 \
flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \
enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \
dangle dtransform reserved cross_read freespace mf vds file_image \
unregister cache_logging cork swmr thread_id vol timer event_set
unregister cache_logging cork swmr thread_id vol timer event_set \
onion
# List programs to be built when testing here
#

4966
test/onion.c Normal file
View File

File diff suppressed because it is too large Load Diff

17
tools/lib/h5diff.c
View File

@ -648,8 +648,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
*-------------------------------------------------------------------------
*/
/* open file 1 */
if (opts->vfd_info[0].u.name) {
if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[0]))) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
}
}
if (opts->custom_vol[0] || opts->custom_vfd[0]) {
if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
if ((fapl1_id = h5tools_get_fapl(fapl1_id, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
@ -664,9 +671,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
H5TOOLS_DEBUG("file1_id = %s", fname1);
/* open file 2 */
if (opts->vfd_info[1].u.name) {
if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
}
}
if (opts->custom_vol[1] || opts->custom_vfd[1]) {
if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
if ((fapl2_id = h5tools_get_fapl(fapl2_id, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");

8
tools/lib/h5tools.c
View File

@ -85,6 +85,7 @@ const char *drivernames[] = {
[WINDOWS_VFD_IDX] = "windows", [STDIO_VFD_IDX] = "stdio", [CORE_VFD_IDX] = "core",
[FAMILY_VFD_IDX] = "family", [SPLIT_VFD_IDX] = "split", [MULTI_VFD_IDX] = "multi",
[MPIO_VFD_IDX] = "mpio", [ROS3_VFD_IDX] = "ros3", [HDFS_VFD_IDX] = "hdfs",
[ONION_VFD_IDX] = "onion",
};
#define NUM_VOLS (sizeof(volnames) / sizeof(volnames[0]))
@ -573,6 +574,13 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled");
#endif
}
else if (!HDstrcmp(vfd_info->u.name, drivernames[ONION_VFD_IDX])) {
/* Onion driver */
if (!vfd_info->info)
H5TOOLS_GOTO_ERROR(FAIL, "Onion VFD info is invalid");
if (H5Pset_fapl_onion(fapl_id, (const H5FD_onion_fapl_info_t *)vfd_info->info) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_onion() failed");
}
else {
/*
* Try to load VFD plugin.

1
tools/lib/h5tools.h
View File

@ -600,6 +600,7 @@ typedef enum {
MPIO_VFD_IDX,
ROS3_VFD_IDX,
HDFS_VFD_IDX,
ONION_VFD_IDX,
} driver_idx;
/* The following include, h5tools_str.h, must be after the

58
tools/src/h5diff/h5diff_common.c
View File

@ -56,6 +56,28 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'},
{"vfd-info-2", require_arg, 'Z'},
{NULL, 0, '\0'}};
static H5FD_onion_fapl_info_t onion_fa_g_1 = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"first input file", /* comment */
};
static H5FD_onion_fapl_info_t onion_fa_g_2 = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"second input file", /* comment */
};
/*-------------------------------------------------------------------------
* Function: check_options
*
@ -451,7 +473,7 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
case '8':
opts->vfd_info[0].type = VFD_BY_NAME;
opts->vfd_info[0].u.name = H5_optarg;
opts->custom_vol[0] = TRUE;
opts->custom_vfd[0] = TRUE;
break;
case '9':
@ -476,6 +498,40 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
}
}
/* If file 1 uses the onion VFD, get the revision number */
if (opts->vfd_info[0].u.name && !HDstrcmp(opts->vfd_info[0].u.name, "onion")) {
if (opts->vfd_info[0].info) {
errno = 0;
onion_fa_g_1.revision_num = HDstrtoull(opts->vfd_info[0].info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for file 1\n");
usage();
h5diff_exit(EXIT_FAILURE);
}
}
else
onion_fa_g_1.revision_num = 0;
opts->vfd_info[0].info = &onion_fa_g_1;
}
/* If file 2 uses the onion VFD, get the revision number */
if (opts->vfd_info[1].u.name && !HDstrcmp(opts->vfd_info[1].u.name, "onion")) {
if (opts->vfd_info[1].info) {
errno = 0;
onion_fa_g_2.revision_num = HDstrtoull(opts->vfd_info[1].info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for file 2\n");
usage();
h5diff_exit(EXIT_FAILURE);
}
}
else
onion_fa_g_2.revision_num = 0;
opts->vfd_info[1].info = &onion_fa_g_2;
}
/* check options */
check_options(opts);

61
tools/src/h5dump/h5dump.c
View File

@ -23,10 +23,13 @@ static hbool_t doxml_g = FALSE;
static hbool_t useschema_g = TRUE;
static const char *xml_dtd_uri_g = NULL;
static hbool_t use_custom_vol_g = FALSE;
static hbool_t use_custom_vfd_g = FALSE;
static h5tools_vol_info_t vol_info_g = {0};
static h5tools_vfd_info_t vfd_info_g = {0};
static hbool_t use_custom_vol_g = FALSE;
static hbool_t use_custom_vfd_g = FALSE;
static h5tools_vol_info_t vol_info_g = {0};
static h5tools_vfd_info_t vfd_info_g = {0};
static hbool_t get_onion_revision_count = FALSE;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
@ -51,6 +54,17 @@ static H5FD_hdfs_fapl_t hdfs_fa_g = {
};
#endif /* H5_HAVE_LIBHDFS */
static H5FD_onion_fapl_info_t onion_fa_g = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"input file", /* comment */
};
/* module-scoped variables for XML option */
#define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
#define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@ -1305,6 +1319,29 @@ end_collect:
}
}
/* If the file uses the onion VFD, get the revision number */
if (vfd_info_g.u.name && !HDstrcmp(vfd_info_g.u.name, "onion")) {
if (vfd_info_g.info) {
if (!HDstrcmp(vfd_info_g.info, "revision_count"))
get_onion_revision_count = TRUE;
else {
errno = 0;
onion_fa_g.revision_num = HDstrtoull(vfd_info_g.info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified\n");
goto error;
}
HDprintf("Using revision %" PRIu64 "\n", onion_fa_g.revision_num);
}
}
else
onion_fa_g.revision_num = 0;
vfd_info_g.info = &onion_fa_g;
}
parse_end:
/* check for file name to be processed */
if (argc <= H5_optind) {
@ -1424,7 +1461,21 @@ main(int argc, char *argv[])
while (H5_optind < argc) {
fname = HDstrdup(argv[H5_optind++]);
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
/* A short cut to get the revision count of an onion file without opening the file */
if (get_onion_revision_count && H5FD_ONION == H5Pget_driver(fapl_id)) {
uint64_t revision_count = 0;
if (H5FDonion_get_revision_count(fname, fapl_id, &revision_count) < 0) {
error_msg("unable to create FAPL for file access\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
HDprintf("The number of revisions for the onion file is %" PRIu64 "\n", revision_count);
goto done;
}
else
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);

28
tools/src/h5repack/h5repack_main.c
View File

@ -76,6 +76,17 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
{"dst-vfd-info", require_arg, 'Z'},
{NULL, 0, '\0'}};
static H5FD_onion_fapl_info_t onion_fa_in_g = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"input file", /* comment */
};
/*-------------------------------------------------------------------------
* Function: usage
*
@ -875,6 +886,23 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
ret_value = -1;
}
/* If the input file uses the onion VFD, get the revision number */
if (in_vfd_info.u.name && !HDstrcmp(in_vfd_info.u.name, "onion")) {
if (in_vfd_info.info) {
errno = 0;
onion_fa_in_g.revision_num = HDstrtoull(in_vfd_info.info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for the input file\n");
usage(h5tools_getprogname());
exit(EXIT_FAILURE);
}
}
else
onion_fa_in_g.revision_num = 0;
in_vfd_info.info = &onion_fa_in_g;
}
/* Setup FAPL for input and output file accesses */
if (custom_in_vol || custom_in_vfd) {
if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL,

23
tools/test/h5diff/CMakeTests.cmake
View File

@ -70,6 +70,13 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_strings2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps1.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps2.h5
# onion VFD files
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5.onion
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5.onion
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5.onion
# tools/testfiles/vds
${HDF5_TOOLS_DIR}/testfiles/vds/1_a.h5
${HDF5_TOOLS_DIR}/testfiles/vds/1_b.h5
@ -295,6 +302,9 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_801.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_830.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_90.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_900.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_901.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_902.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8625.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8639.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_reg.txt
@ -932,6 +942,12 @@
h5diff_8639.out.err
h5diff_90.out
h5diff_90.out.err
h5diff_900.out
h5diff_900.out.err
h5diff_901.out
h5diff_901.out.err
h5diff_902.out
h5diff_902.out.err
h5diff_v1.out
h5diff_v1.out.err
h5diff_v2.out
@ -1563,6 +1579,13 @@ ADD_H5_TEST (h5diff_v1 0 -v ${FILEV1} ${FILEV2})
ADD_H5_TEST (h5diff_v2 0 -r ${FILEV1} ${FILEV2})
ADD_H5_TEST (h5diff_v3 0 -c ${FILEV1} ${FILEV2})
# ##############################################################################
# # onion VFD tests (serial only)
# ##############################################################################
ADD_SH5_TEST (h5diff_900 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5)
ADD_SH5_TEST (h5diff_901 0 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5)
ADD_SH5_TEST (h5diff_902 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5)
##############################################################################
### P L U G I N T E S T S
##############################################################################

729
tools/test/h5diff/h5diffgentest.c
View File

@ -38,30 +38,34 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024);
*-------------------------------------------------------------------------
*/
#define FILE1 "h5diff_basic1.h5"
#define FILE2 "h5diff_basic2.h5"
#define FILE3 "h5diff_types.h5"
#define FILE4 "h5diff_dtypes.h5"
#define FILE5 "h5diff_attr1.h5"
#define FILE6 "h5diff_attr2.h5"
#define FILE6a "h5diff_attr3.h5"
#define FILE7 "h5diff_dset1.h5"
#define FILE8 "h5diff_dset2.h5"
#define FILE8A "h5diff_dset3.h5"
#define FILE9 "h5diff_hyper1.h5"
#define FILE10 "h5diff_hyper2.h5"
#define FILE11 "h5diff_empty.h5"
#define FILE12 "h5diff_links.h5"
#define FILE13 "h5diff_softlinks.h5"
#define FILE14 "h5diff_linked_softlink.h5"
#define FILE15 "h5diff_extlink_src.h5"
#define FILE16 "h5diff_extlink_trg.h5"
#define FILE17 "h5diff_ext2softlink_src.h5"
#define FILE18 "h5diff_ext2softlink_trg.h5"
#define FILE19 "h5diff_dset_zero_dim_size1.h5"
#define FILE20 "h5diff_dset_zero_dim_size2.h5"
#define FILE21 "h5diff_dset_idx1.h5"
#define FILE22 "h5diff_dset_idx2.h5"
#define FILE1 "h5diff_basic1.h5"
#define FILE2 "h5diff_basic2.h5"
#define FILE3 "h5diff_types.h5"
#define FILE4 "h5diff_dtypes.h5"
#define FILE5 "h5diff_attr1.h5"
#define FILE6 "h5diff_attr2.h5"
#define FILE6a "h5diff_attr3.h5"
#define FILE7 "h5diff_dset1.h5"
#define FILE8 "h5diff_dset2.h5"
#define FILE8A "h5diff_dset3.h5"
#define FILE9 "h5diff_hyper1.h5"
#define FILE10 "h5diff_hyper2.h5"
#define FILE11 "h5diff_empty.h5"
#define FILE12 "h5diff_links.h5"
#define FILE13 "h5diff_softlinks.h5"
#define FILE14 "h5diff_linked_softlink.h5"
#define FILE15 "h5diff_extlink_src.h5"
#define FILE16 "h5diff_extlink_trg.h5"
#define FILE17 "h5diff_ext2softlink_src.h5"
#define FILE18 "h5diff_ext2softlink_trg.h5"
#define FILE19 "h5diff_dset_zero_dim_size1.h5"
#define FILE20 "h5diff_dset_zero_dim_size2.h5"
#define FILE21 "h5diff_dset_idx1.h5"
#define FILE22 "h5diff_dset_idx2.h5"
#define FILE23 "h5diff_onion_dset_1d.h5"
#define FILE24 "h5diff_onion_objs.h5"
#define FILE25 "h5diff_onion_dset_ext.h5"
#define DANGLE_LINK_FILE1 "h5diff_danglelinks1.h5"
#define DANGLE_LINK_FILE2 "h5diff_danglelinks2.h5"
#define GRP_RECURSE_FILE1 "h5diff_grp_recurse1.h5"
@ -111,6 +115,11 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024);
#define SPACE1_DIM1 0
#define SPACE1_DIM2 0
/* For Onion VFD */
#define ONION_TEST_FIXNAME_SIZE 1024
#define ONION_TEST_PAGE_SIZE (uint32_t)32
#define ONE_DIM_SIZE 16
/* Error macros */
#define AT() HDprintf("ERROR at %s:%d in %s()...\n", __FILE__, __LINE__, __func__);
#define PROGRAM_ERROR \
@ -179,6 +188,11 @@ static void test_objs_nocomparables(const char *fname1, const char *fname2);
static void test_objs_strings(const char *fname, const char *fname2);
static void test_double_epsilon(const char *fname1, const char *fname2);
/* Generate the files for testing Onion VFD */
static int test_onion_1d_dset(const char *fname);
static int test_onion_create_delete_objects(const char *fname);
static int test_onion_dset_extension(const char *fname);
/* called by test_attributes() and test_datasets() */
static void write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs);
static void write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs);
@ -298,9 +312,676 @@ main(void)
/* double dataset and epsilion. HDFFV-10897 */
test_double_epsilon(DIFF_EPS1, DIFF_EPS2);
/* Generate the files for testing Onion VFD */
test_onion_1d_dset(FILE23);
test_onion_create_delete_objects(FILE24);
test_onion_dset_extension(FILE25);
return EXIT_SUCCESS;
}
/* Structure to collect the onion filepaths in one place. */
struct onion_filepaths {
char *canon;
char *onion;
char *recovery;
};
/* Allocate and populate filepaths with h5_fixname'd strings as appropriate.
* Should be released with onion_filepaths_destroy() when done.
*/
static struct onion_filepaths *
onion_filepaths_init(const char *basename)
{
struct onion_filepaths *paths = NULL;
if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths))))
goto error;
if (NULL == (paths->canon = HDstrdup(basename)))
goto error;
if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon);
if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon);
return paths;
error:
if (paths != NULL) {
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
HDfree(paths);
}
return NULL;
}
static void
onion_filepaths_destroy(struct onion_filepaths *s)
{
if (s) {
HDfree(s->canon);
HDfree(s->onion);
HDfree(s->recovery);
HDfree(s);
}
}
static int
test_onion_1d_dset(const char *fname)
{
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {1, ONE_DIM_SIZE}, maxdims[2] = {1, ONE_DIM_SIZE};
int wdata[1][ONE_DIM_SIZE], /* Write buffer */
fillval;
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file_id = H5I_INVALID_HID;
/* Setup */
onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata[0][i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0)
goto error;
/* Close and release resources */
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
int dset_data[1][ONE_DIM_SIZE];
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + 2048;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
goto error;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data[0][i] = i + 3072;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(onion_info.backing_fapl_id);
H5Pclose(fapl_id);
H5Dclose(dset);
H5Sclose(space);
H5Fclose(file_id);
}
H5E_END_TRY;
return -1;
} /* test_onion_1d_dset */
static int
test_onion_create_delete_objects(const char *fname)
{
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t fapl_id = H5I_INVALID_HID;
hid_t group_id = H5I_INVALID_HID;
hid_t attr_space_id = H5I_INVALID_HID;
hid_t attr_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t attr_dim[1] = {4};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int wdata[4][4]; /* Write buffer */
int fillval;
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/*
* Initialize data.
*/
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/*
* Create a new file using the default properties.
*/
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/*
* Create dataspace with unlimited dimensions.
*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/*
* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/*
* Set the fill value for the dataset.
*/
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/*
* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and add an attribute to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Create dataspace for attribute */
attr_space_id = H5Screate_simple(1, attr_dim, NULL);
if ((attr_id =
H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Sclose(attr_space_id) < 0)
goto error;
if (H5Aclose(attr_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fourth revision: open the file with Onion VFD and delete the attribute
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Adelete(file, "file_attribute") < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fifth revision: open the file with Onion VFD and add a group to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Gclose(group_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Sixth revision: open the file with Onion VFD and delete the newly added group
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*
* Close and release resources.
*/
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
if (dset != H5I_INVALID_HID)
(void)H5Dclose(dset);
if (file != H5I_INVALID_HID)
(void)H5Fclose(file);
if (fapl_id != H5I_INVALID_HID)
(void)H5Pclose(fapl_id);
if (onion_info.backing_fapl_id != H5I_INVALID_HID)
H5Pclose(onion_info.backing_fapl_id);
return -1;
} /* test_onion_create_delete_objects */
static int
test_onion_dset_extension(const char *fname)
{
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset_space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4}, maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, chunk[2] = {4, 4};
hsize_t size[2], offset[2];
int wdata[4][4], /* Write buffer */
fillval;
/* Setup */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and extend the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and double the rows */
size[0] = 2 * dims[0];
size[1] = dims[1];
if (H5Dset_extent(dset, size) < 0)
goto error;
if ((dset_space = H5Dget_space(dset)) < 0)
goto error;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
goto error;
/* Write the data to the dataset. */
if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Sclose(dset_space) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and shrink the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and shrink back the size */
if (H5Dset_extent(dset, dims) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/* Close and release resources. */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* test_onion_dset_extension */
/*-------------------------------------------------------------------------
* Function: test_basic
*

12
tools/test/h5diff/testfiles/h5diff_900.txt Normal file
View File

@ -0,0 +1,12 @@
file1 file2
---------------------------------------
x x /
x x /DS1
x /DS2
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
0 differences found
EXIT CODE: 1

17
tools/test/h5diff/testfiles/h5diff_901.txt Normal file
View File

@ -0,0 +1,17 @@
file1 file2
---------------------------------------
x x /
x x /DS1
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
Not comparable: </DS1> has rank 2, dimensions [4x4], max dimensions [18446744073709551615x18446744073709551615]
and </DS1> has rank 2, dimensions [8x4], max dimensions [18446744073709551615x18446744073709551615]
0 differences found
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects without details of differences.
EXIT CODE: 0

30
tools/test/h5diff/testfiles/h5diff_902.txt Normal file
View File

@ -0,0 +1,30 @@
file1 file2
---------------------------------------
x x /
x x /DS1
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
size: [1x16] [1x16]
position DS1 DS1 difference
------------------------------------------------------------
[ 0 0 ] 0 16 16
[ 0 1 ] 1 17 16
[ 0 2 ] 2 18 16
[ 0 3 ] 3 19 16
[ 0 4 ] 4 20 16
[ 0 5 ] 5 21 16
[ 0 6 ] 6 22 16
[ 0 7 ] 7 23 16
[ 0 8 ] 8 24 16
[ 0 9 ] 9 25 16
[ 0 10 ] 10 26 16
[ 0 11 ] 11 27 16
[ 0 12 ] 12 28 16
[ 0 13 ] 13 29 16
[ 0 14 ] 14 30 16
[ 0 15 ] 15 31 16
16 differences found
EXIT CODE: 1

BIN
tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5.onion Normal file
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5.onion Normal file
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_onion_objs.h5 Normal file
View File

Binary file not shown.

BIN
tools/test/h5diff/testfiles/h5diff_onion_objs.h5.onion Normal file
View File

Binary file not shown.

18
tools/test/h5diff/testh5diff.sh.in
View File

@ -124,6 +124,12 @@ $SRC_H5DIFF_TESTFILES/h5diff_strings1.h5
$SRC_H5DIFF_TESTFILES/h5diff_strings2.h5
$SRC_H5DIFF_TESTFILES/h5diff_eps1.h5
$SRC_H5DIFF_TESTFILES/h5diff_eps2.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5.onion
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5.onion
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5.onion
$SRC_TOOLS_TESTFILES/tvlstr.h5
"
@ -356,6 +362,9 @@ $SRC_H5DIFF_TESTFILES/h5diff_800.txt
$SRC_H5DIFF_TESTFILES/h5diff_801.txt
$SRC_H5DIFF_TESTFILES/h5diff_830.txt
$SRC_H5DIFF_TESTFILES/h5diff_90.txt
$SRC_H5DIFF_TESTFILES/h5diff_900.txt
$SRC_H5DIFF_TESTFILES/h5diff_901.txt
$SRC_H5DIFF_TESTFILES/h5diff_902.txt
$SRC_H5DIFF_TESTFILES/h5diff_8625.txt
$SRC_H5DIFF_TESTFILES/h5diff_8639.txt
$SRC_H5DIFF_TESTFILES/h5diff_reg.txt
@ -1214,6 +1223,15 @@ TOOLTEST h5diff_v1.txt -v 1_vds.h5 2_vds.h5
TOOLTEST h5diff_v2.txt -r 1_vds.h5 2_vds.h5
TOOLTEST h5diff_v3.txt -c 1_vds.h5 2_vds.h5
# ##############################################################################
# Onion VFD tests
# ##############################################################################
# These tests won't pass under ph5diff
if test -z "$pmode"; then
TOOLTEST h5diff_900.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5
TOOLTEST h5diff_901.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5
TOOLTEST h5diff_902.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5
fi
# ##############################################################################
# # END

20
tools/test/h5dump/CMakeTests.cmake
View File

@ -211,6 +211,11 @@
${HDF5_TOOLS_DIR}/testfiles/trefer_paramR.ddl
${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1dR.ddl
${HDF5_TOOLS_DIR}/testfiles/trefer_regR.ddl
# Onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_revision_count.ddl
)
set (HDF5_N_REFERENCE_FILES
tall-3
@ -350,7 +355,13 @@
${HDF5_TOOLS_DIR}/testfiles/trefer_param.h5
${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1d.h5
${HDF5_TOOLS_DIR}/testfiles/trefer_reg.h5
# Onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion
)
set (HDF5_ERROR_REFERENCE_TEST_FILES
${PROJECT_SOURCE_DIR}/errfiles/filter_fail.err
@ -1187,6 +1198,13 @@
ADD_H5_TEST (tCVE_2018_11206_fill_old 1 tCVE_2018_11206_fill_old.h5)
ADD_H5_TEST (tCVE_2018_11206_fill_new 1 tCVE_2018_11206_fill_new.h5)
# onion VFD tests
ADD_H5_TEST (tst_onion_objs 0 --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5)
ADD_H5_TEST (tst_onion_dset_ext 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5)
ADD_H5_TEST (tst_onion_dset_1d 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5)
ADD_H5_TEST (tst_onion_revision_count 0 --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5)
##############################################################################
### P L U G I N T E S T S
##############################################################################

677
tools/test/h5dump/h5dumpgentest.c
View File

@ -117,6 +117,13 @@
#define FILE87 "tintsnodata.h5"
#define FILE88 "tldouble_scalar.h5"
#define FILE89 "tfloatsattrs.h5"
#define FILE90 "tst_onion_dset_1d.h5"
#define FILE91 "tst_onion_objs.h5"
#define FILE92 "tst_onion_dset_ext.h5"
#define ONION_TEST_FIXNAME_SIZE 1024
#define ONION_TEST_PAGE_SIZE (uint32_t)32
#define ONE_DIM_SIZE 16
/*-------------------------------------------------------------------------
* prototypes
@ -11350,6 +11357,671 @@ error:
H5E_END_TRY;
} /* gen_err_attr_dspace() */
/* Structure to collect the onion filepaths in one place. */
struct onion_filepaths {
char *canon;
char *onion;
char *recovery;
};
/* Allocate and populate filepaths with h5_fixname'd strings as appropriate.
* Should be released with onion_filepaths_destroy() when done.
*/
static struct onion_filepaths *
onion_filepaths_init(const char *basename)
{
struct onion_filepaths *paths = NULL;
if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths))))
goto error;
if (NULL == (paths->canon = HDstrdup(basename)))
goto error;
if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon);
if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon);
return paths;
error:
if (paths != NULL) {
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
HDfree(paths);
}
return NULL;
}
static void
onion_filepaths_destroy(struct onion_filepaths *s)
{
if (s) {
HDfree(s->canon);
HDfree(s->onion);
HDfree(s->recovery);
HDfree(s);
}
}
static int
gent_onion_1d_dset(void)
{
hid_t file_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hsize_t dims[2] = {1, ONE_DIM_SIZE};
hsize_t maxdims[2] = {1, ONE_DIM_SIZE};
int wdata[1][ONE_DIM_SIZE];
int fillval;
/* Setup */
onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE90)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata[0][i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0)
goto error;
/* Close and release resources */
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
int dset_data[1][ONE_DIM_SIZE];
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + 2048;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
goto error;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data[0][i] = i + 3072;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(onion_info.backing_fapl_id);
H5Pclose(fapl_id);
H5Dclose(dset);
H5Sclose(space);
H5Fclose(file_id);
}
H5E_END_TRY;
return -1;
} /* gent_onion_1d_dset */
static int
gent_onion_create_delete_objects(void)
{
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t fapl_id = H5I_INVALID_HID;
hid_t group_id = H5I_INVALID_HID;
hid_t attr_space_id = H5I_INVALID_HID;
hid_t attr_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t attr_dim[1] = {4};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int wdata[4][4];
int fillval;
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE91)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/*
* Initialize data.
*/
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/*
* Create a new file using the default properties.
*/
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/*
* Create dataspace with unlimited dimensions.
*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/*
* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/*
* Set the fill value for the dataset.
*/
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/*
* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and add an attribute to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Create dataspace for attribute */
attr_space_id = H5Screate_simple(1, attr_dim, NULL);
if ((attr_id =
H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Sclose(attr_space_id) < 0)
goto error;
if (H5Aclose(attr_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fourth revision: open the file with Onion VFD and delete the attribute
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Adelete(file, "file_attribute") < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fifth revision: open the file with Onion VFD and add a group to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Gclose(group_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Sixth revision: open the file with Onion VFD and delete the newly added group
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*
* Close and release resources.
*/
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
if (dset != H5I_INVALID_HID)
(void)H5Dclose(dset);
if (file != H5I_INVALID_HID)
(void)H5Fclose(file);
if (fapl_id != H5I_INVALID_HID)
(void)H5Pclose(fapl_id);
if (onion_info.backing_fapl_id != H5I_INVALID_HID)
H5Pclose(onion_info.backing_fapl_id);
return -1;
} /* gent_onion_create_delete_objects */
static int
gent_onion_dset_extension(void)
{
hid_t fapl_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset_space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
hsize_t size[2];
hsize_t offset[2];
int wdata[4][4]; /* Write buffer */
int fillval;
/* Setup */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE92)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and extend the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and double the rows */
size[0] = 2 * dims[0];
size[1] = dims[1];
if (H5Dset_extent(dset, size) < 0)
goto error;
if ((dset_space = H5Dget_space(dset)) < 0)
goto error;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
goto error;
/* Write the data to the dataset. */
if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Sclose(dset_space) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and shrink the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and shrink back the size */
if (H5Dset_extent(dset, dims) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/* Close and release resources. */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* gent_onion_dset_extension */
int
main(void)
{
@ -11449,5 +12121,10 @@ main(void)
gent_err_attr_dspace();
/* Generate the files for testing Onion VFD */
gent_onion_1d_dset();
gent_onion_create_delete_objects();
gent_onion_dset_extension();
return 0;
}

16
tools/test/h5dump/testh5dump.sh.in
View File

@ -184,6 +184,12 @@ $SRC_H5DUMP_TESTFILES/tvms.h5
$SRC_H5DUMP_TESTFILES/err_attr_dspace.h5
$SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_old.h5
$SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_new.h5
$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5
$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5.onion
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5.onion
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5.onion
"
LIST_OTHER_TEST_FILES="
@ -372,6 +378,10 @@ $SRC_H5DUMP_TESTFILES/h5dump-help.txt
$SRC_H5DUMP_TESTFILES/out3.h5import
$SRC_H5DUMP_TESTFILES/tbinregR.exp
$SRC_H5DUMP_TESTFILES/err_attr_dspace.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_objs.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_revision_count.ddl
"
LIST_ERROR_TEST_FILES="
@ -1486,6 +1496,12 @@ TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5
TOOLTEST_FAIL tCVE_2018_11206_fill_old.h5
TOOLTEST_FAIL tCVE_2018_11206_fill_new.h5
# test Onion VFD
TOOLTEST tst_onion_objs.ddl --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5
TOOLTEST tst_onion_dset_ext.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5
TOOLTEST tst_onion_dset_1d.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5
TOOLTEST tst_onion_revision_count.ddl --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR

55
tools/test/h5repack/CMakeTests.cmake
View File

@ -122,6 +122,13 @@
${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5
# tools/testfiles onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion
)
set (LIST_OTHER_TEST_FILES
@ -446,6 +453,50 @@
endif ()
endmacro ()
macro (ADD_H5_DMP_NO_OPT_TEST testname testtype resultcode resultfile)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_DMP-${testname}
COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}"
)
set_property(TEST H5REPACK_DMP-${testname} PROPERTY DISABLED)
endif ()
else ()
add_test (
NAME H5REPACK_DMP-${testname}-clear-objects
COMMAND ${CMAKE_COMMAND} -E remove testfiles/out-${testname}.${resultfile}
)
set_tests_properties (H5REPACK_DMP-${testname}-clear-objects PROPERTIES
FIXTURES_REQUIRED clear_h5repack
)
add_test (
NAME H5REPACK_DMP-${testname}
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_file_ext}> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
)
set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES
DEPENDS H5REPACK_DMP-${testname}-clear-objects
)
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_DMP-h5dump-${testname}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5dump${tgt_file_ext}>"
-D "TEST_ARGS:STRING=out-${testname}.${resultfile}"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
-D "TEST_OUTPUT=${resultfile}-${testname}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${testname}.${resultfile}.ddl"
-P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
)
set_tests_properties (H5REPACK_DMP-h5dump-${testname} PROPERTIES
DEPENDS "H5REPACK_DMP-${testname}"
)
endif ()
endif ()
endmacro ()
macro (ADD_H5_DIFF_TEST testname testtype resultcode testfile)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
@ -1665,6 +1716,10 @@ ADD_H5_DMP_TEST (textlink-mergeprune "TEST" 0 textlink.h5 --merge --prune --enab
### HDFFV-11128 needs fixed to enable the following test
#ADD_H5_DMP_TEST (textlinktar-mergeprune "TEST" 0 textlinktar.h5 --merge --prune --enable-error-stack)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_1d "TEST" 0 tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_ext "TEST" 0 tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_objs "TEST" 0 tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1)
##############################################################################
### P L U G I N T E S T S
##############################################################################

64
tools/test/h5repack/h5repack.sh.in
View File

@ -177,6 +177,13 @@ $SRC_TOOLS_TESTFILES/vds/5_a.h5
$SRC_TOOLS_TESTFILES/vds/5_b.h5
$SRC_TOOLS_TESTFILES/vds/5_c.h5
$SRC_TOOLS_TESTFILES/vds/5_vds.h5
########tools/testfiles########
$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5
$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5.onion
$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5
$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5.onion
$SRC_TOOLS_TESTFILES/tst_onion_objs.h5
$SRC_TOOLS_TESTFILES/tst_onion_objs.h5.onion
"
LIST_OTHER_TEST_FILES="
@ -251,6 +258,10 @@ $SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
########onion#files########
$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_1d.h5.ddl
$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_ext.h5.ddl
$SRC_H5REPACK_TESTFILES/onion.tst_onion_objs.h5.ddl
"
#
@ -1130,6 +1141,56 @@ TOOLTEST_DUMP()
rm -f $outfile
}
# This is same as TOOLTEST_DUMP() with comparing h5dump output
# without any option
#
TOOLTEST_DUMP_NO_OPT()
{
infile=$2
outfile=out-$1.$2
expect="$TESTDIR/$1.$2.ddl"
actual="$TESTDIR/out-$1.$2.out"
actual_err="$TESTDIR/out-$1.$2.err"
shift
shift
# Run test.
TESTING $H5REPACK $@
(
cd $TESTDIR
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
) >$actual 2>$actual_err
RET=$?
if [ $RET != 0 ] ; then
echo "*FAILED*"
nerrors="`expr $nerrors + 1`"
else
echo " PASSED"
VERIFY h5dump output $@
(
cd $TESTDIR
$RUNSERIAL $H5DUMP_BIN $outfile
) >$actual 2>$actual_err
cat $actual_err >> $actual
RET=$?
fi
if cmp -s $expect $actual; then
echo " PASSED"
else
echo "*FAILED*"
echo " Expected result (*.ddl) differs from actual result (*.out)"
nerrors="`expr $nerrors + 1`"
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
fi
rm -f $actual $actual_err
rm -f $outfile
}
# This is similar to TOOLTEST_DUMP().
# Test h5repack with options added for paged aggregation.
# h5stat is used on the repacked file and the expected output
@ -1830,6 +1891,9 @@ TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-sta
#TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack
TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1
TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1
TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR

11
tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl Normal file
View File

@ -0,0 +1,11 @@
HDF5 "out-onion.tst_onion_dset_1d.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) }
DATA {
(0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
}
}
}
}

18
tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl Normal file
View File

@ -0,0 +1,18 @@
HDF5 "out-onion.tst_onion_dset_ext.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6,
(4,0): 0, 1, 2, 3,
(5,0): 1, 2, 3, 4,
(6,0): 2, 3, 4, 5,
(7,0): 3, 4, 5, 6
}
}
}
}

24
tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl Normal file
View File

@ -0,0 +1,24 @@
HDF5 "out-onion.tst_onion_objs.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
DATASET "DS2" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
}
}

12
tools/testfiles/tst_onion_dset_1d.ddl Normal file
View File

@ -0,0 +1,12 @@
Using revision 1
HDF5 "tst_onion_dset_1d.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) }
DATA {
(0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
}
}
}
}

BIN
tools/testfiles/tst_onion_dset_1d.h5 Normal file
View File

Binary file not shown.

BIN
tools/testfiles/tst_onion_dset_1d.h5.onion Normal file
View File

Binary file not shown.

19
tools/testfiles/tst_onion_dset_ext.ddl Normal file
View File

@ -0,0 +1,19 @@
Using revision 1
HDF5 "tst_onion_dset_ext.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6,
(4,0): 0, 1, 2, 3,
(5,0): 1, 2, 3, 4,
(6,0): 2, 3, 4, 5,
(7,0): 3, 4, 5, 6
}
}
}
}

BIN
tools/testfiles/tst_onion_dset_ext.h5 Normal file
View File

Binary file not shown.

BIN
tools/testfiles/tst_onion_dset_ext.h5.onion Normal file
View File

Binary file not shown.

22
tools/testfiles/tst_onion_objs.ddl Normal file
View File

@ -0,0 +1,22 @@
Using revision 3
HDF5 "tst_onion_objs.h5" {
GROUP "/" {
ATTRIBUTE "file_attribute" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4 ) / ( 4 ) }
DATA {
(0): 0, 0, 0, 0
}
}
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
}
}

BIN
tools/testfiles/tst_onion_objs.h5 Normal file
View File

Binary file not shown.

BIN
tools/testfiles/tst_onion_objs.h5.onion Normal file
View File

Binary file not shown.

1
tools/testfiles/tst_onion_revision_count.ddl Normal file
View File

@ -0,0 +1 @@
The number of revisions for the onion file is 6