diff --git a/.gitignore b/.gitignore index f042b35a1..2273a6bc5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ scan-build .deps .libs *.zip -*.gz Makefile .DS_Store build-par diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5337ff15f..238740753 100755 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,6 +5,10 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. +## 4.4.1 - TBD + +* Addressed an issue where `ncdump` would crash when trying to read a netcdf file containing an empty ragged `VLEN` variable in an unlimited dimension. See [GitHub #221](https://github.com/Unidata/netcdf-c/issues/221) for more information. + ## 4.4.0 Released - January 13, 2016 * Bumped SO version to 11.0.0. diff --git a/gh221.nc.gz b/gh221.nc.gz new file mode 100644 index 000000000..e7adbb404 Binary files /dev/null and b/gh221.nc.gz differ diff --git a/libsrc4/nc4hdf.c b/libsrc4/nc4hdf.c index f90401aed..2edfb4de6 100644 --- a/libsrc4/nc4hdf.c +++ b/libsrc4/nc4hdf.c @@ -1147,6 +1147,7 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp, filldata = (char *)data + real_data_size; for (i = 0; i < fill_len; i++) { + if (var->type_info->nc_type_class == NC_STRING) { if (*(char **)fillvalue) @@ -1157,7 +1158,13 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp, else *(char **)filldata = NULL; } - else + else if(var->type_info->nc_type_class == NC_VLEN) { + if(fillvalue) { + memcpy(filldata,fillvalue,file_type_size); + } else { + *(char **)filldata = NULL; + } + } else memcpy(filldata, fillvalue, file_type_size); filldata = (char *)filldata + file_type_size; } diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 28a066ba6..566891210 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -15,13 +15,20 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4 tst_vars build_bin_test(renamegroup) add_sh_test(nc_test4 run_grp_rename) +## +# The shell script, run_empty_vlen_test.sh, +# depends on the 'tst_empty_vlen_unlim' binary. +## +BUILD_BIN_TEST(tst_empty_vlen_unlim) +ADD_SH_TEST(nc_test4 run_empty_vlen_test) + + IF(NOT MSVC) SET(NC4_TESTS ${NC4_TESTS} tst_interops5 tst_camrun) ENDIF() # If the v2 API was built, add the test program. IF(ENABLE_V2_API) - build_bin_test(tst_v2) ENDIF() @@ -82,8 +89,3 @@ IF(TEST_PARALLEL) build_bin_test(tst_mode) add_sh_test(nc_test4 run_par_test) ENDIF() - -## Specify files to be distributed by 'make dist' -FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} ref_chunks1.cdl ref_chunks2.cdl ref_tst_compounds.nc ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc ref_tst_dims.nc ref_tst_interops4.nc ref_grp_rename.cdl ref_tst_nvars.nc contig.hdf4 chunked.hdf4) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 5d03046a7..0ce5717bc 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -23,7 +23,7 @@ t_type cdm_sea_soundings tst_camrun tst_vl tst_atts1 tst_atts2 \ tst_vars2 tst_files5 tst_files6 tst_sync tst_h_strbug tst_h_refs \ tst_h_scalar tst_rename tst_h5_endians tst_atts_string_rewrite -check_PROGRAMS = $(NC4_TESTS) renamegroup +check_PROGRAMS = $(NC4_TESTS) renamegroup tst_empty_vlen_unlim # Add these if large file tests are turned on. if LARGE_FILE_TESTS @@ -32,7 +32,7 @@ endif TESTS = $(NC4_TESTS) -TESTS += run_grp_rename.sh +TESTS += run_grp_rename.sh run_empty_vlen_test.sh # If the v2 API was built, add its test program. if BUILD_V2 @@ -123,7 +123,7 @@ ref_tst_interops4.nc run_get_knmi_files.sh CMakeLists.txt \ run_grp_rename.sh tst_formatx_hdf4.sh \ run_chunk_hdf4.sh contiguous.hdf4 chunked.hdf4 \ tst_h5_endians.c tst_h4_lendian.c tst_atts_string_rewrite.c \ -tst_put_vars_two_unlim_dim.c +tst_put_vars_two_unlim_dim.c tst_empty_vlen_unlim.c run_empty_vlen_test.sh CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt \ @@ -134,7 +134,7 @@ tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* \ usi_01.* thetau_01.* tst_*.nc tst_*.h5 \ tst_grp_rename.cdl tst_grp_rename.nc tst_grp_rename.dmp ref_grp_rename.cdl \ foo1.nc tst_interops2.h4 tst_h5_endians.nc tst_h4_lendian.h4 test.nc \ -tst_atts_string_rewrite.nc +tst_atts_string_rewrite.nc tst_empty_vlen_unlim.nc tst_empty_vlen_lim.nc if USE_HDF4_FILE_TESTS DISTCLEANFILES = AMSR_E_L2_Rain_V10_200905312326_A.hdf \ diff --git a/nc_test4/run_empty_vlen_test.sh b/nc_test4/run_empty_vlen_test.sh new file mode 100755 index 000000000..adddd29a2 --- /dev/null +++ b/nc_test4/run_empty_vlen_test.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Front-end for tst_empty_vlen_unlim.c. This test +# ensures that valid netcdf files are generated when +# a ragged VLEN array is defined but not populated. +# +# This script runs tst_empty_vlen_unlim and then +# runs `ncdump` against the two generated files. +# +# See https://github.com/Unidata/netcdf-c/issues/221 for +# full details. +# + +set -e + +if test "x$srcdir" = x ; then +srcdir=`pwd` +fi + +echo "" +echo "* Testing Empty Ragged Arrays (VLEN)" + +echo "Generating test netcdf files." +./tst_empty_vlen_unlim + +echo "Validating Files with ncdump." +echo "======================================" +../ncdump/ncdump -s tst_empty_vlen_unlim.nc +echo "---------------------------------------" +../ncdump/ncdump -s tst_empty_vlen_lim.nc +echo "======================================" + +echo "* Tests Passed." +exit 0 diff --git a/nc_test4/tst_empty_vlen_unlim.c b/nc_test4/tst_empty_vlen_unlim.c new file mode 100644 index 000000000..63f15356a --- /dev/null +++ b/nc_test4/tst_empty_vlen_unlim.c @@ -0,0 +1,224 @@ +/* This is part of the netCDF package. Copyright 2005 University + Corporation for Atmospheric Research/Unidata See COPYRIGHT file for + conditions of use. + + This program excersizes HDF5 variable length array code. + + $Id: tst_h_vl2.c,v 1.5 2010/06/01 15:34:52 ed Exp $ +*/ + +/* This test was added to help diagnose the issue described at: + * * https://github.com/Unidata/netcdf-c/issues/221 + * This issue was originally reported by the python group at: + * * https://github.com/Unidata/netcdf4-python/issues/527 + */ + + +#include +#include +#include + +#define FILE_NAME_UNLIM "tst_empty_vlen_unlim.nc" +#define FILE_NAME_LIM "tst_empty_vlen_lim.nc" +#define DIM_LEN_UNLIM NC_UNLIMITED +#define DIM_LEN_LIM 5 +#define DIM_NAME "x" +#define VLEN_NAME "vltest" +#define VAR_NAME1 "v" +#define VAR_NAME2 "w" +#define ROW_COUNT 3 +#define VLEN0 2 +#define VLEN1 3 +#define VLEN2 3 + +int main() { + + printf("Testing access to unset entries in VLEN variable, unlimited dimension\n"); + { + int ncid, typeid, dimid, varid, varid2; + nc_vlen_t data[ROW_COUNT]; + int stat; + float *dat0, *dat1, *dat2; + float *data2; + size_t startp[3] = {0,0,0}; + size_t countp[3] = {VLEN0,VLEN1,VLEN2}; + size_t startp2[1] = {0}; + size_t countp2[1] = {VLEN2}; + int i = 0; + /* Create File */ + printf("\t* Creating File:\tnc_create()\n"); + if (nc_create(FILE_NAME_UNLIM, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Set fill mode */ + //printf("\t* Setting fill mode:\tnc_set_fill()\n"); + //if(nc_set_fill(ncid,NC_FILL,NULL)) ERR; + + /* Create Dimension */ + printf("\t* Defining Unlimited Dimension:\tnc_def_dim()\n"); + if (nc_def_dim(ncid, DIM_NAME, DIM_LEN_UNLIM, &dimid)) ERR; + + /* Create ragged array type. */ + printf("\t* Creating Ragged Array type:\tnc_def_vlen().\n"); + if (nc_def_vlen(ncid, VLEN_NAME, NC_FLOAT, &typeid)) ERR; + + /* Create a variable of typeid. */ + printf("\t* Creating Variable using Ragged Arrayt Type:\tnc_def_var().\n"); + if (nc_def_var(ncid, VAR_NAME1, typeid, 1, &dimid, &varid)) ERR; + + /* Create a variable of type float. */ + printf("\t* Creating secondary Variable using NC_FLOAT:\tnc_def_var().\n"); + if (nc_def_var(ncid, VAR_NAME2, NC_FLOAT, 1, &dimid, &varid2)) ERR; + + /* End define mode. */ + printf("\t* Ending define mode:\tnc_enddef().\n"); + + /* Write out data for w */ + printf("\t* Creating float data for secondary variable.\n"); + data2 = (float*)malloc(sizeof(float) * VLEN2); + for(i = 0; i < VLEN2; i++) { + data2[i] = (float)i; + } + + printf("\t* Puting data in secondary variable:\tnc_put_vara().\n"); + if (nc_put_vara(ncid,varid2,&startp2,&countp2,data2)) ERR; + + /***********/ + /* Actually unnecessary to recreate the issue. */ + /***********/ + + + /* Write out varying-length data for v[0] and v[1]. Leave v[2] empty. */ + + dat0 = (float*)malloc(VLEN0 * sizeof(float)); + for(i = 0; i < VLEN0; i++) { + dat0[i] = (float)i; + } + dat1 = (float*)malloc(VLEN1 * sizeof(float)); + for(i = 0; i < VLEN1; i++) { + dat1[i] = (float)i; + } + dat2 = (float*)malloc(VLEN2 * sizeof(float)); + for(i = 0; i < VLEN2; i++) { + dat2[i] = (float)i; + } + + data[0].p = dat0; + data[0].len = VLEN0; + + data[1].p = dat1; + data[1].len = VLEN1; + + data[2].p = dat2; + data[2].len = VLEN2; + + //printf("\t* Puting data in VLEN variable:\tnc_put_vara().\n"); + //stat = nc_put_vara(ncid,varid,&startp,&countp,data); + //stat = nc_put_var(ncid,varid,&data); + //if(stat) ERR; + + + + /* Close File. */ + printf("\t* Closing file:\tnc_close().\n"); + if (stat = nc_close(ncid)) ERR; + + + } + + printf("Testing access to unset entries in VLEN variable, unlimit dimension\n"); + { + int ncid, typeid, dimid, varid, varid2; + nc_vlen_t data[ROW_COUNT]; + int stat; + float *dat0, *dat1, *dat2; + float *data2; + size_t startp[3] = {0,0,0}; + size_t countp[3] = {VLEN0,VLEN1,VLEN2}; + size_t startp2[1] = {0}; + size_t countp2[1] = {VLEN2}; + int i = 0; + /* Create File */ + printf("\t* Creating File:\tnc_create()\n"); + if (nc_create(FILE_NAME_LIM, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Set fill mode */ + //printf("\t* Setting fill mode:\tnc_set_fill()\n"); + //if(nc_set_fill(ncid,NC_FILL,NULL)) ERR; + + /* Create Dimension */ + printf("\t* Defining Unlimited Dimension:\tnc_def_dim()\n"); + if (nc_def_dim(ncid, DIM_NAME, DIM_LEN_LIM, &dimid)) ERR; + + /* Create ragged array type. */ + printf("\t* Creating Ragged Array type:\tnc_def_vlen().\n"); + if (nc_def_vlen(ncid, VLEN_NAME, NC_FLOAT, &typeid)) ERR; + + /* Create a variable of typeid. */ + printf("\t* Creating Variable using Ragged Arrayt Type:\tnc_def_var().\n"); + if (nc_def_var(ncid, VAR_NAME1, typeid, 1, &dimid, &varid)) ERR; + + /* Create a variable of type float. */ + printf("\t* Creating secondary Variable using NC_FLOAT:\tnc_def_var().\n"); + if (nc_def_var(ncid, VAR_NAME2, NC_FLOAT, 1, &dimid, &varid2)) ERR; + + /* End define mode. */ + printf("\t* Ending define mode:\tnc_enddef().\n"); + + /* Write out data for w */ + printf("\t* Creating float data for secondary variable.\n"); + data2 = (float*)malloc(sizeof(float) * VLEN2); + for(i = 0; i < VLEN2; i++) { + data2[i] = (float)i; + } + + printf("\t* Puting data in secondary variable:\tnc_put_vara().\n"); + if (nc_put_vara(ncid,varid2,&startp2,&countp2,data2)) ERR; + + /***********/ + /* Actually unnecessary to recreate the issue. */ + /***********/ + + + /* Write out varying-length data for v[0] and v[1]. Leave v[2] empty. */ + + dat0 = (float*)malloc(VLEN0 * sizeof(float)); + for(i = 0; i < VLEN0; i++) { + dat0[i] = (float)i; + } + dat1 = (float*)malloc(VLEN1 * sizeof(float)); + for(i = 0; i < VLEN1; i++) { + dat1[i] = (float)i; + } + dat2 = (float*)malloc(VLEN2 * sizeof(float)); + for(i = 0; i < VLEN2; i++) { + dat2[i] = (float)i; + } + + data[0].p = dat0; + data[0].len = VLEN0; + + data[1].p = dat1; + data[1].len = VLEN1; + + data[2].p = dat2; + data[2].len = VLEN2; + + //printf("\t* Puting data in VLEN variable:\tnc_put_vara().\n"); + //stat = nc_put_vara(ncid,varid,&startp,&countp,data); + //stat = nc_put_var(ncid,varid,&data); + //if(stat) ERR; + + + + /* Close File. */ + printf("\t* Closing file:\tnc_close().\n"); + if (stat = nc_close(ncid)) ERR; + + + } + + + SUMMARIZE_ERR; + FINAL_RESULTS; + +}