netcdf-c/libhdf4/hdf4file.c

/* Copyright 2018, UCAR/Unidata See netcdf/COPYRIGHT file for copying
 * and redistribution conditions.*/
/**
 * @file @internal The HDF4 file functions.
 *
 * @author Ed Hartnett
 */

#include "config.h"
#include "nc4internal.h"
#include "hdf4dispatch.h"
#include <mfhdf.h>

#define NUM_TYPES 12 /**< Number of netCDF atomic types. */

extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);

/** @internal These flags may not be set for open mode. */
static const int
ILLEGAL_OPEN_FLAGS = (NC_MMAP|NC_64BIT_OFFSET|NC_DISKLESS|NC_WRITE);

/** @internal NetCDF atomic type names. */
static const char*
nc_type_name_g[NUM_TYPES] = {"char", "byte", "short", "int", "float", "double",
                             "ubyte", "ushort", "uint", "int64", "uint64",
                             "string"};

/** @internal NetCDF atomic type sizes. */
static const size_t
nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof(short),
                             sizeof(int), sizeof(float), sizeof(double),
                             sizeof(unsigned char), sizeof(unsigned short),
                             sizeof(unsigned int), sizeof(long long),
                             sizeof(unsigned long long), sizeof(char *)};

/**
 * @internal Recursively delete the data for a group (and everything
 * it contains) in our internal metadata store.
 *
 * @param grp Pointer to group info struct.
 *
 * @return ::NC_NOERR No error.
 * @author Ed Hartnett
 */
static int
hdf4_rec_grp_del(NC_GRP_INFO_T *grp)
{
   NC_VAR_INFO_T *var;
   int i;

   assert(grp);
   LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));

   /* Delete all vars. */
   for (i = 0; i < ncindexsize(grp->vars); i++)
   {
      NC_VAR_HDF4_INFO_T *hdf4_var;
      
      var = (NC_VAR_INFO_T *)ncindexith(grp->vars, i);
      assert(var);
      LOG((4, "%s: deleting var %s", __func__, var->hdr.name));

      /* Get the HDF4 specific var metadata. */
      hdf4_var = (NC_VAR_HDF4_INFO_T *)var->format_var_info;

      /* Close HDF4 dataset associated with this var, unless it's a
       * scale. */
      if (hdf4_var->sdsid && SDendaccess(hdf4_var->sdsid) < 0)
         return NC_EHDFERR;
   }

   return NC_NOERR;
}

/**
 * @internal Given an HDF4 type, set a pointer to netcdf type.
 *
 * See http://www.hdfgroup.org/training/HDFtraining/UsersGuide/Fundmtls.fm3.html
 * for more information re: HDF4 types.
 *
 * @param h5 Pointer to HDF5 file info struct.
 * @param hdf4_typeid Type ID for hdf4 datatype.
 * @param xtype Pointer that gets netcdf type. Ignored if NULL.
 * @param endniannessp Pointer that gets endianness. Ignored if NULL.
 * @param type_sizep Pointer that gets type size. Ignored if NULL.
 * @param type_name Pointer that gets the type name. Ignored if NULL.
 *
 * @return ::NC_NOERR No error.
 * @author Ed Hartnett
 */
static int
hdf4_type_info(NC_FILE_INFO_T *h5, int32 hdf4_typeid, nc_type* xtypep,
               int *endiannessp, size_t *type_sizep, char *type_name)
{
   int t = 0;
   int endianness = NC_ENDIAN_BIG;
   nc_type xtype;

   assert(h5);

   switch(hdf4_typeid)
   {
   case DFNT_CHAR:
      xtype = NC_CHAR;
      t = 0;
      break;
   case DFNT_UCHAR:
   case DFNT_UINT8:
      xtype = NC_UBYTE;
      t = 6;
      break;
   case DFNT_LUINT8:
      xtype = NC_UBYTE;
      t = 6;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_INT8:
      xtype = NC_BYTE;
      t = 1;
      break;
   case DFNT_LINT8:
      xtype = NC_BYTE;
      t = 1;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_INT16:
      xtype = NC_SHORT;
      t = 2;
      break;
   case DFNT_LINT16:
      xtype = NC_SHORT;
      t = 2;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_UINT16:
      xtype = NC_USHORT;
      t = 7;
      break;
   case DFNT_LUINT16:
      xtype = NC_USHORT;
      t = 7;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_INT32:
      xtype = NC_INT;
      t = 3;
      break;
   case DFNT_LINT32:
      xtype = NC_INT;
      t = 3;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_UINT32:
      xtype = NC_UINT;
      t = 8;
      break;
   case DFNT_LUINT32:
      xtype = NC_UINT;
      t = 8;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_FLOAT32:
      xtype = NC_FLOAT;
      t = 4;
      break;
   case DFNT_LFLOAT32:
      xtype = NC_FLOAT;
      t = 4;
      endianness = NC_ENDIAN_LITTLE;
      break;
   case DFNT_FLOAT64:
      xtype = NC_DOUBLE;
      t = 5;
      break;
   case DFNT_LFLOAT64:
      xtype = NC_DOUBLE;
      t = 5;
      endianness = NC_ENDIAN_LITTLE;
      break;
   default:
      return NC_EBADTYPID;
   }

   /* Return results to caller. */
   if (xtypep)
      *xtypep = xtype;
   if (endiannessp)
      *endiannessp = endianness;
   if (type_sizep)
      *type_sizep = nc_type_size_g[t];
   if (type_name)
      strncpy(type_name, nc_type_name_g[t], NC_MAX_NAME);

   return NC_NOERR;
}

/**
 * @internal Set the type of a netCDF-4 variable.
 *
 * @param xtype A netcdf type.
 * @param endianness The endianness of the data.
 * @param type_size The size in bytes of one element of this type.
 * @param type_name A name for the type.
 * @param typep Pointer to a pointer that gets the TYPE_INFO_T struct.
 *
 * @return ::NC_NOERR No error.
 * @author Ed Hartnett
 */
static int
nc4_set_var_type(nc_type xtype, int endianness, size_t type_size, char *type_name,
                 NC_TYPE_INFO_T **typep)
{
   NC_TYPE_INFO_T *type;

   /* Check inputs. */
   assert(typep);

   /* Allocate space for the type info struct. */
   if (!(type = calloc(1, sizeof(NC_TYPE_INFO_T))))
      return NC_ENOMEM;
   if (!(type->hdr.name = strdup(type_name)))
   {
      free(type);
      return NC_ENOMEM;
   }
   type->hdr.sort = NCTYP;

   /* Determine the type class. */
   if (xtype == NC_FLOAT)
      type->nc_type_class = NC_FLOAT;
   else if (xtype == NC_DOUBLE)
      type->nc_type_class = NC_DOUBLE;
   else if (xtype == NC_CHAR)
      type->nc_type_class = NC_STRING;
   else
      type->nc_type_class = NC_INT;

   /* Set other type info values. */
   type->endianness = endianness;
   type->size = type_size;
   type->hdr.id = (size_t)xtype;
   type->hdr.hashkey = NC_hashmapkey(type->hdr.name, strlen(type->hdr.name));

   /* Return to caller. */
   *typep = type;

   return NC_NOERR;
}

/**
 * @internal Read an attribute from a HDF4 file.
 *
 * @param h5 Pointer to the file metadata struct.
 * @param var Pointer to variable metadata struct or NULL for global
 * attributes.
 * @param a Index of attribute to read.
 *
 * @return ::NC_NOERR No error.
 * @return ::NC_EHDFERR HDF4 error.
 * @return ::NC_EATTMETA Error reading HDF4 attribute.
 * @return ::NC_ENOMEM Out of memory.
 * @author Ed Hartnett
 */
static int
hdf4_read_att(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, int a)
{
   NC_HDF4_FILE_INFO_T *hdf4_file;
   NC_ATT_INFO_T *att;   
   NCindex *att_list;   
   int32 att_data_type, att_count;
   size_t att_type_size;
   char name[NC_MAX_HDF4_NAME+1];
   int sd_id;
   int retval;
   
   LOG((3, "%s: a %d var %s", __func__, a, var ? var->hdr.name : "global"));

   /* Check inputs. */
   assert(h5 && h5->format_file_info);

   /* Get the HDF4 file info. */
   hdf4_file = h5->format_file_info;

   /* Decide what att list to use, global or from a var. */
   if (var)
   {
      NC_VAR_HDF4_INFO_T *hdf4_var;
      assert(var->format_var_info);
      att_list = var->att;
      hdf4_var = var->format_var_info;
      sd_id = hdf4_var->sdsid;
   } else {
      att_list = h5->root_grp->att;
      sd_id = hdf4_file->sdid;
   }

   /* Learn about this attribute. */
   if (SDattrinfo(sd_id, a, name, &att_data_type, &att_count))
      return NC_EATTMETA;
   
   /* Get information about the attribute type. */
   nc_type xtype;
   if ((retval = hdf4_type_info(h5, att_data_type, &xtype, NULL,
                                &att_type_size, NULL)))
      return retval;

   /* Add to the end of the list of atts for this var. */
   if ((retval = nc4_att_list_add(att_list, name, &att)))
      return retval;
   att->nc_typeid = xtype;
   att->created = NC_TRUE;
   att->len = att_count;
   
   /* Allocate memory to hold the data. */
   if (att->len)
      if (!(att->data = malloc(att_type_size * att->len)))
         return NC_ENOMEM;
   
   /* Read the data. */
   if (SDreadattr(sd_id, a, att->data))
      return NC_EHDFERR;

   return NC_NOERR;
}

/**
 * @internal Read a HDF4 dimension. As new dimensions are found, add
 * them to the metadata list of dimensions.
 *
 * @param h5 Pointer to the file metadata struct.
 * @param var Pointer to variable metadata struct or NULL for global
 * attributes.
 * @param rec_dim_len Actual length of first dim for this SD.
 * @param d Dimension index for this SD.
 * 
 * @return ::NC_NOERR No error.
 * @return ::NC_EHDFERR HDF4 error.
 * @return ::NC_EDIMMETA Error reading HDF4 dimension info.
 * @return ::NC_ENOMEM Out of memory.
 * @return ::NC_EMAXNAME Name too long.
 * @author Ed Hartnett
 */
static int
hdf4_read_dim(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, int rec_dim_len, int d)
{
   NC_VAR_HDF4_INFO_T *hdf4_var;
   NC_DIM_INFO_T *dim = NULL;
   int32 dimid, dim_len, dim_data_type, dim_num_attrs;
   char dim_name[NC_MAX_NAME + 1];
   int i;
   int retval;
   
   assert(h5 && h5->format_file_info && var && var->format_var_info);
   hdf4_var = var->format_var_info;         

   /* Get information about the dimension. */
   if ((dimid = SDgetdimid(hdf4_var->sdsid, d)) == FAIL)
      return NC_EDIMMETA;
   if (SDdiminfo(dimid, dim_name, &dim_len, &dim_data_type, &dim_num_attrs))
      return NC_EDIMMETA;
   if (strlen(dim_name) > NC_MAX_HDF4_NAME)
      return NC_EMAXNAME;
   
   /* Do we already have this dimension? HDF4 explicitly uses
    * the name to tell. */
   for (i = 0; i < ncindexsize(h5->root_grp->dim); i++)
   {
      dim = (NC_DIM_INFO_T*)ncindexith(h5->root_grp->dim, i);
      if (!strcmp(dim->hdr.name, dim_name))
         break;
      dim = NULL;
   }
   
   /* If we didn't find this dimension, add one. */
   if (!dim)
   {
      LOG((4, "adding dim %s for dataset %s", dim_name, var->hdr.name));
      if ((retval = nc4_dim_list_add(h5->root_grp, dim_name,
                                     (dim_len ? dim_len : rec_dim_len), -1, &dim)))
         return retval;
   }
   
   /* Tell the variable the id of this dimension. */
   var->dimids[d] = dim->hdr.id;
   var->dim[d] = dim;

   return NC_NOERR;
}

/**
 * @internal Create a new variable and insert int relevant lists
 *
 * @param grp the containing group
 * @param name the name for the new variable
 * @param ndims the rank of the new variable
 * @param format_var_info Pointer to format-specific var info struct.
 * @param var Pointer in which to return a pointer to the new var.
 *
 * @return ::NC_NOERR No error.
 * @return ::NC_ENOMEM Out of memory.
 * @author Ed Hartnett
 */
static int
nc4_var_list_add_full(NC_GRP_INFO_T* grp, const char* name, int ndims, nc_type xtype,
                      int endianness, size_t type_size, char *type_name, void *fill_value,
                      int contiguous, size_t *chunksizes, void *format_var_info,
                      NC_VAR_INFO_T **var)
{
   int d;
   int retval;

   /* Add the VAR_INFO_T struct to our list of vars. */
   if ((retval = nc4_var_list_add(grp, name, ndims, var)))
      return retval;
   (*var)->created = NC_TRUE;
   (*var)->written_to = NC_TRUE;
   (*var)->format_var_info = format_var_info;
   (*var)->atts_read = 1;

   /* Fill special type_info struct for variable type information. */
   if ((retval = nc4_set_var_type(xtype, endianness, type_size, type_name,
                                  &(*var)->type_info)))
      return retval;

   (*var)->type_info->rc++;
   
   /* Handle fill value, if provided. */
   if (fill_value)
   {
      if (!((*var)->fill_value = malloc(type_size)))
         return NC_ENOMEM;
      memcpy((*var)->fill_value, fill_value, type_size);
   }

   /* Var contiguous or chunked? */
   (*var)->contiguous = contiguous;
   
   /* Were chunksizes provided? */
   if (chunksizes)
   {
      if (!((*var)->chunksizes = malloc(ndims * sizeof(size_t))))
         return NC_ENOMEM;
      for (d = 0; d < ndims; d++) 
         (*var)->chunksizes[d] = chunksizes[d];
   }

   return NC_NOERR;
}

/**
 * @internal Read a HDF4 variable, including its associated dimensions
 * and attributes.
 *
 * @param h5 Pointer to the file metadata struct.
 * @param v Index of variable to read.
 * 
 * @return ::NC_NOERR No error.
 * @return ::NC_EHDFERR HDF4 error.
 * @return ::NC_EDIMMETA Error reading HDF4 dimension info.
 * @return ::NC_EVARMETA Error reading HDF4 dataset or att.
 * @return ::NC_EATTMETA Error reading HDF4 attribute.
 * @return ::NC_ENOMEM Out of memory.
 * @return ::NC_EMAXNAME Name too long.
 * @author Ed Hartnett
 */
static int
hdf4_read_var(NC_FILE_INFO_T *h5, int v)
{
   NC_HDF4_FILE_INFO_T *hdf4_file;
   NC_VAR_INFO_T *var;
   NC_VAR_HDF4_INFO_T *hdf4_var;
   HDF_CHUNK_DEF chunkdefs;
   int32 data_type, num_atts;
   int32 dimsize[NC_MAX_HDF4_DIMS];
   int32 rec_dim_len;
   int32 rank;
   int32 sdsid;
   int contiguous;
   int d, a;
   int flag;
   char name[NC_MAX_HDF4_NAME+1];
   int xtype;
   char type_name[NC_MAX_NAME + 1];
   int endianness;
   size_t type_size;
   void *fill_value;
   size_t *chunksizes = NULL;
   int retval;

   /* Check inputs. */
   assert(h5 && h5->format_file_info);

   /* Get HDF4 file metadata. */
   hdf4_file = h5->format_file_info;

   /* Open this dataset in HDF4 file. */
   if ((sdsid = SDselect(hdf4_file->sdid, v)) == FAIL)
      return  NC_EVARMETA;

   /* Learn about this dataset. */
   if (SDgetinfo(sdsid, name, &rank, dimsize, &data_type, &num_atts))
      return NC_EVARMETA;
   rec_dim_len = dimsize[0];

   /* Get chunking info from HDF4 file. */
   if (SDgetchunkinfo(sdsid, &chunkdefs, &flag))
      return NC_EVARMETA;

   /* Learn about the HDF4 type. */
   if ((retval = hdf4_type_info(h5, data_type, &xtype, &endianness, &type_size,
                                type_name)))
      return retval;

   /* Get the fill value. */
   if (!(fill_value = malloc(type_size)))
      return NC_ENOMEM;
   if (SDgetfillvalue(sdsid, fill_value))
   {
      /* Whoops! No fill value! */
      free(fill_value);
      fill_value = NULL;
   }

   /* Is variable chunked or contiguous? */
   if (flag == HDF_NONE)
      contiguous = NC_TRUE;
   else if (flag & HDF_CHUNK)
   {
      contiguous = NC_FALSE;
      if (!(chunksizes = malloc(rank * sizeof(size_t))))
         return NC_ENOMEM;
      for (d = 0; d < rank; d++) 
         chunksizes[d] = chunkdefs.chunk_lengths[d];
   }

   /* Malloc a struct to hold HDF4-specific variable
    * information. */
   if (!(hdf4_var = malloc(sizeof(NC_VAR_HDF4_INFO_T))))
      return NC_ENOMEM;

   /* Remember these values. */
   hdf4_var->hdf4_data_type = data_type;
   hdf4_var->sdsid = sdsid;   

   /* Add a variable to metadata structures. */
   LOG((3, "adding var for HDF4 dataset %s, rank %d netCDF type %d", name,
        rank, xtype));
   retval = nc4_var_list_add_full(h5->root_grp, name, (int)rank,
                                  xtype, endianness, type_size, type_name,                                       
                                  fill_value, contiguous, chunksizes, hdf4_var,
                                  &var);

   /* Free resources. */
   if (chunksizes)
      free(chunksizes);
   if (fill_value)
      free(fill_value);

   /* Did the add fail? */
   if (retval)
   {
      free(hdf4_var);
      return retval;
   }

   /* Find the variable's dimensions. */
   for (d = 0; d < var->ndims; d++)
      if ((retval = hdf4_read_dim(h5, var, rec_dim_len, d)))
         return retval;

   /* Read the variable's attributes. */
   for (a = 0; a < num_atts; a++)
      if ((retval = hdf4_read_att(h5, var, a)))
         return retval;

   return NC_NOERR;
}

/**
 * @internal Open a HDF4 file.
 *
 * @param path The file name of the file.
 * @param mode The open mode flag.
 * @param basepe Ignored by this function.
 * @param chunksizehintp Ignored by this function.
 * @param parameters pointer to struct holding extra data (e.g. for
 * parallel I/O) layer. Ignored if NULL.
 * @param dispatch Pointer to the dispatch table for this file.
 * @param nc_file Pointer to an instance of NC.
 *
 * @return ::NC_NOERR No error.
 * @return ::NC_EINVAL Invalid input.
 * @author Ed Hartnett
 */
int
NC_HDF4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
             void *parameters, NC_Dispatch *dispatch, NC *nc_file)
{
   NC_FILE_INFO_T *h5;
   NC_HDF4_FILE_INFO_T *hdf4_file;
   int32 num_datasets, num_gatts;
   int32 sdid;
   int v, a;
   NC_FILE_INFO_T* nc4_info = NULL;
   int retval = NC_NOERR;

   /* Check inputs. */
   assert(nc_file && path);

   LOG((1, "%s: path %s mode %d params %x", __func__, path, mode, parameters));

   /* Check the mode for validity */
   if (mode & ILLEGAL_OPEN_FLAGS)
      return NC_EINVAL;

   /* Open the file. */
   nc_file->int_ncid = nc_file->ext_ncid;

   /* Open the file and initialize SD interface. */
   if ((sdid = SDstart(path, DFACC_READ)) == FAIL)
      return NC_EHDFERR;

   /* Learn how many datasets and global atts we have. */
   if (SDfileinfo(sdid, &num_datasets, &num_gatts))
      return NC_EHDFERR;

   /* Add necessary structs to hold netcdf-4 file data. */
   if ((retval = nc4_nc4f_list_add(nc_file, path, mode)))
      return retval;
   nc4_info = NC4_DATA(nc_file);
   assert(nc4_info && nc4_info->root_grp);
   h5 = nc4_info;
   h5->no_write = NC_TRUE;
   h5->root_grp->atts_read = 1;

   /* Allocate data to hold HDF4 specific file data. */
   if (!(hdf4_file = malloc(sizeof(NC_HDF4_FILE_INFO_T))))
      return NC_ENOMEM;
   h5->format_file_info = hdf4_file;
   hdf4_file->sdid = sdid;   

   /* Read the global atts. */
   for (a = 0; a < num_gatts; a++)
      if ((retval = hdf4_read_att(h5, NULL, a)))
         break;

   /* Read each dataset. */
   if (!retval)
      for (v = 0; v < num_datasets; v++)
         if ((retval = hdf4_read_var(h5, v)))
            break;

   /* If there is an error, free resources. */
   if (retval)
      free(hdf4_file);
   
#ifdef LOGGING
   /* This will print out the names, types, lens, etc of the vars and
      atts in the file, if the logging level is 2 or greater. */
   log_metadata_nc(h5);
#endif

   return retval;
}

/**
 * @internal Abort (close) the HDF4 file.
 *
 * @param ncid File ID.
 *
 * @return ::NC_NOERR No error.
 * @return ::NC_EBADID Bad ncid.
 * @author Ed Hartnett
 */
int
NC_HDF4_abort(int ncid)
{
   return NC_HDF4_close(ncid, NULL);
}

/**
 * @internal Close the HDF4 file.
 *
 * @param ncid File ID.
 * @param ignore Ignore this pointer.
 *
 * @return ::NC_NOERR No error.
 * @return ::NC_EBADID Bad ncid.
 * @author Ed Hartnett
 */
int
NC_HDF4_close(int ncid, void *ignore)
{
   NC_GRP_INFO_T *grp;
   NC *nc;
   NC_FILE_INFO_T *h5;
   NC_HDF4_FILE_INFO_T *hdf4_file;
   int retval;

   LOG((1, "%s: ncid 0x%x", __func__, ncid));

   /* Find our metadata for this file. */
   if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
      return retval;
   assert(nc && h5 && grp && !grp->parent);

   /* Clean up HDF4 specific allocations. */
   if ((retval = hdf4_rec_grp_del(h5->root_grp)))
      return retval;

   /* Delete all the list contents for vars, dims, and atts, in each
    * group. */
   if ((retval = nc4_rec_grp_del(h5->root_grp)))
      return retval;

   /* Close hdf4 file and free HDF4 file info. */
   hdf4_file = (NC_HDF4_FILE_INFO_T *)h5->format_file_info;
   if (SDend(hdf4_file->sdid))
      return NC_EHDFERR;
   free(hdf4_file);

   /* Misc. Cleanup */
   nclistfree(h5->alldims);
   nclistfree(h5->allgroups);
   nclistfree(h5->alltypes);

   /* Free the nc4_info struct; above code should have reclaimed
      everything else */
   free(h5);

   return NC_NOERR;
}