Merge pull request #2138 from DennisHeimbigner/fixosbugs.dmh

Fix a number of OS specific bugs
This commit is contained in:
Ward Fisher 2021-11-04 13:36:49 -06:00 committed by GitHub
commit 33e4ec4675
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 161 additions and 143 deletions

View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.2 - TBD
* [Bug Fix] Fix several os related errors. See [Github #2138](https://github.com/Unidata/netcdf-c/pull/2138).
* [Enhancement] Support byte-range reading of netcdf-3 files stored in private buckets in S3. See [Github #2134](https://github.com/Unidata/netcdf-c/pull/2134)
* [Enhancement] Support Amazon S3 access for NCZarr. Also support use of the existing Amazon SDK credentials system. See [Github #2114](https://github.com/Unidata/netcdf-c/pull/2114)
* [Bug Fix] Fix string allocation error in H5FDhttp.c. See [Github #2127](https://github.com/Unidata/netcdf-c/pull/2127).

View File

@ -1170,14 +1170,17 @@ case "`uname`" in
CYGWIN*) ISCYGWIN=yes;;
Darwin*) ISOSX=yes;;
WIN*) ISMSVC=yes;;
MINGW*) ISMINGW=yes;;
esac
AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes])
AM_CONDITIONAL(ISOSX, [test "x$ISOSX" = xyes])
AM_CONDITIONAL(ISMINGW, [test "x$ISMINGW" = xyes])
AC_SUBST([ISMSVC], [${ISMSVC}])
AC_SUBST([ISCYGWIN], [${ISCYGWIN}])
AC_SUBST([ISOSX], [${ISOSX}])
AC_SUBST([ISMINGW], [${ISMINGW}])
###
# Crude hack to work around an issue

View File

@ -60,9 +60,6 @@ extern long long int strtoll(const char*, char**, int);
#ifndef strtoull
extern unsigned long long int strtoull(const char*, char**, int);
#endif
#ifndef fileno
extern int fileno(FILE*);
#endif
#endif /*STDC*/
#endif /*!_WIN32*/

View File

@ -19,6 +19,7 @@ and accessing rc files (e.g. .daprc).
/* getenv() keys */
#define NCRCENVIGNORE "NCRCENV_IGNORE"
#define NCRCENVRC "NCRCENV_RC"
#define NCRCENVHOME "NCRCENV_HOME"
/* Known .aws profile keys */
#define AWS_ACCESS_KEY_ID "aws_access_key_id"
@ -38,6 +39,7 @@ typedef struct NCRCinfo {
int loaded; /* 1 => already loaded */
NClist* entries; /* the rc file entry store fields*/
char* rcfile; /* specified rcfile; overrides anything else */
char* rchome; /* Overrides $HOME when looking for .rc files */
} NCRCinfo;
/* Collect global state info in one place */

View File

@ -871,10 +871,12 @@ getlocalpathkind(void)
int kind = NCPD_UNKNOWN;
#ifdef __CYGWIN__
kind = NCPD_CYGWIN;
#elif __MSYS__
kind = NCPD_MSYS;
#elif _MSC_VER /* not _WIN32 */
#elif defined __MINGW32__
kind = NCPD_WIN;
#elif defined _MSC_VER /* not _WIN32 */
kind = NCPD_WIN;
#elif defined __MSYS__
kind = NCPD_MSYS;
#else
kind = NCPD_NIX;
#endif

View File

@ -127,6 +127,18 @@ ncrc_initialize(void)
}
}
static void
ncrc_setrchome(void)
{
const char* tmp = NULL;
if(ncrc_globalstate->rcinfo.rchome) return;
assert(ncrc_globalstate && ncrc_globalstate->home);
tmp = getenv(NCRCENVHOME);
if(tmp == NULL || strlen(tmp) == 0)
tmp = ncrc_globalstate->home;
ncrc_globalstate->rcinfo.rchome = strdup(tmp);
}
/* Get global state */
NCRCglobalstate*
ncrc_getglobalstate(void)
@ -155,6 +167,7 @@ NC_rcclear(NCRCinfo* info)
{
if(info == NULL) return;
nullfree(info->rcfile);
nullfree(info->rchome);
rcfreeentries(info->entries);
}
@ -193,9 +206,9 @@ NC_rcload(void)
/* locate the configuration files in order of use:
1. Specified by NCRCENV_RC environment variable.
2. If NCRCENV_RC is not set then merge the set of rc files in this order:
1. $HOME/.ncrc
2. $HOME/.daprc
3. $HOME/.docsrc
1. $RCHOME/.ncrc
2. $RCHOME/.daprc
3. $RCHOME/.docsrc
4. $CWD/.ncrc
5. $CWD/.daprc
6. $CWD/.docsrc
@ -208,7 +221,11 @@ NC_rcload(void)
const char* dirnames[3];
const char** dir;
dirnames[0] = globalstate->home;
/* Make sure rcinfo.rchome is defined */
ncrc_setrchome();
dirnames[0] = globalstate->rcinfo.rchome;
dirnames[1] = globalstate->cwd;
dirnames[2] = NULL;

View File

@ -256,6 +256,8 @@ NC_writefile(const char* filename, size_t size, void* content)
void* p;
size_t remain;
if(content == NULL) {content = ""; size = 0;}
#ifdef _WIN32
stream = NCfopen(filename,"wb");
#else
@ -343,7 +345,8 @@ done:
return found;
}
#ifdef __APPLE__
#if ! defined __INTEL_COMPILER
#if defined __APPLE__
int isinf(double x)
{
union { unsigned long long u; double f; } ieee754;
@ -361,6 +364,8 @@ int isnan(double x)
}
#endif /*APPLE*/
#endif /*!_INTEL_COMPILER*/
int
NC_split_delim(const char* arg, char delim, NClist* segments)

View File

@ -28,6 +28,8 @@
#include "nc3internal.h"
#include "netcdf_mem.h"
#include "ncpathmgr.h"
#include "ncrc.h"
#include "ncbytes.h"
#undef DEBUG
@ -710,48 +712,16 @@ readfile(const char* path, NC_memio* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t filesize = 0;
size_t count = 0;
char* memory = NULL;
char* p = NULL;
NCbytes* buf = ncbytesnew();
/* Open the file for reading */
#ifdef _MSC_VER
f = NCfopen(path,"rb");
#else
f = NCfopen(path,"r");
#endif
if(f == NULL)
{status = errno; goto done;}
/* get current filesize */
if(fseek(f,0,SEEK_END) < 0)
{status = errno; goto done;}
filesize = (size_t)ftell(f);
/* allocate memory */
memory = malloc((size_t)filesize);
if(memory == NULL)
{status = NC_ENOMEM; goto done;}
/* move pointer back to beginning of file */
rewind(f);
count = filesize;
p = memory;
while(count > 0) {
size_t actual;
actual = fread(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
}
if((status = NC_readfile(path,buf))) goto done;
if(memio) {
memio->size = (size_t)filesize;
memio->memory = memory;
memory = NULL;
memio->size = ncbyteslength(buf);
memio->memory = ncbytesextract(buf);
}
done:
if(memory != NULL)
free(memory);
ncbytesfree(buf);
if(f != NULL) fclose(f);
return status;
}
@ -761,30 +731,10 @@ static int
writefile(const char* path, NCMEMIO* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t count = 0;
char* p = NULL;
/* Open/create the file for writing*/
#ifdef _MSC_VER
f = NCfopen(path,"wb");
#else
f = NCfopen(path,"w");
#endif
if(f == NULL)
{status = errno; goto done;}
rewind(f);
count = memio->size;
p = memio->memory;
while(count > 0) {
size_t actual;
actual = fwrite(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
if(memio) {
if((status = NC_writefile(path,memio->size,memio->memory))) goto done;
}
done:
if(f != NULL) fclose(f);
return status;
}

View File

@ -160,13 +160,13 @@ endif()
SET_TARGET_PROPERTIES(nctrunc PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
IF(RCMERGE)
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
endif()
IF(RCMERGE)
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
endif()
IF(USE_HDF5)
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY
@ -189,7 +189,6 @@ endif()
add_sh_test(ncdump tst_64bit)
add_bin_test_no_prefix(ref_ctest)
add_bin_test_no_prefix(ref_ctest64)
add_sh_test(ncdump tst_output)
add_sh_test(ncdump tst_lengths)
add_sh_test(ncdump tst_calendars)
build_bin_test_no_prefix(tst_utf8)
@ -200,15 +199,21 @@ endif()
add_sh_test(ncdump tst_hdf5_offset)
ENDIF(USE_HDF5)
IF(NOT MSVC AND NOT MINGW)
add_sh_test(ncdump tst_output)
ENDIF()
add_sh_test(ncdump tst_null_byte_padding)
IF(USE_STRICT_NULL_BYTE_HEADER_PADDING)
SET_TESTS_PROPERTIES(ncdump_tst_null_byte_padding PROPERTIES WILL_FAIL TRUE)
ENDIF(USE_STRICT_NULL_BYTE_HEADER_PADDING)
add_sh_test(ncdump tst_nccopy3)
IF(HAVE_BASH)
SET_TESTS_PROPERTIES(ncdump_tst_nccopy3 PROPERTIES RUN_SERIAL TRUE)
ENDIF(HAVE_BASH)
IF(NOT MSVC AND NOT MINGW)
add_sh_test(ncdump tst_nccopy3)
IF(HAVE_BASH)
SET_TESTS_PROPERTIES(ncdump_tst_nccopy3 PROPERTIES RUN_SERIAL TRUE)
ENDIF(HAVE_BASH)
ENDIF()
add_sh_test(ncdump tst_nccopy3_subset)
add_sh_test(ncdump tst_charfill)
@ -256,10 +261,11 @@ endif()
# formatting omits a 0.
###
IF(EXTRA_TESTS)
add_sh_test(ncdump run_back_comp_tests)
IF(MSVC)
SET_TESTS_PROPERTIES(ncdump_run_back_comp_tests PROPERTIES WILL_FAIL TRUE)
ENDIF(MSVC)
IF(USE_HDF5)
IF(NOT MSVC AND NOT MINGW)
add_sh_test(ncdump run_back_comp_tests)
ENDIF()
ENDIF()
ENDIF(EXTRA_TESTS)
# Known failure on MSVC; the number of 0's padding
@ -272,10 +278,11 @@ endif()
build_bin_test_no_prefix(tst_fillbug)
add_sh_test(ncdump_sh tst_fillbug)
add_sh_test(ncdump tst_netcdf4_4)
IF(MSVC AND HAVE_BASH)
SET_TESTS_PROPERTIES(ncdump_tst_netcdf4_4 PROPERTIES WILL_FAIL TRUE)
ENDIF(MSVC AND HAVE_BASH)
IF(HAVE_BASH)
IF(NOT MSVC AND NOT MINGW)
add_sh_test(ncdump tst_netcdf4_4)
ENDIF()
ENDIF(HAVE_BASH)
###
# Some test reordering was required to ensure these tests
@ -324,11 +331,11 @@ endif()
IF(USE_HDF5)
IF(HAVE_BASH)
build_bin_test_no_prefix(tst_unicode)
IF(NOT MSVC)
IF(NOT MSVC AND NOT MINGW)
# These tests do not work under windows
add_sh_test(ncdump test_unicode_directory)
add_sh_test(ncdump test_unicode_path)
ENDIF(NOT MSVC)
ENDIF()
ENDIF(HAVE_BASH)
ENDIF(USE_HDF5)

View File

@ -76,10 +76,11 @@ bom tst_dimsizes nctrunc tst_rcmerge
# Tests for classic and 64-bit offset files.
TESTS = tst_inttags.sh run_tests.sh tst_64bit.sh ref_ctest \
ref_ctest64 tst_output.sh tst_lengths.sh tst_calendars.sh \
run_utf8_tests.sh tst_nccopy3.sh tst_nccopy3_subset.sh \
ref_ctest64 tst_lengths.sh tst_calendars.sh \
run_utf8_tests.sh tst_nccopy3_subset.sh \
tst_charfill.sh tst_iter.sh tst_formatx3.sh tst_bom.sh \
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh #test_rcmerge.sh
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh \
test_radix.sh test_rcmerge.sh
# The tst_nccopy3.sh test uses output from a bunch of other
# tests. This records the dependency so parallel builds work.
@ -91,10 +92,6 @@ if USE_STRICT_NULL_BYTE_HEADER_PADDING
XFAIL_TESTS += tst_null_byte_padding.sh
endif
if ! ISCYGWIN
TESTS += test_unicode_directory.sh test_unicode_path.sh
endif
if LARGE_FILE_TESTS
TESTS += tst_iter.sh
endif
@ -113,9 +110,9 @@ check_PROGRAMS += tst_vlen_demo
# Tests for netCDF-4 behavior.
TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \
tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \
tst_netcdf4.sh tst_fillbug.sh tst_nccopy4.sh \
tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \
run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \
run_utf8_nc4_tests.sh run_ncgen_nc4_tests.sh \
tst_ncgen4.sh test_scope.sh
# Record interscript dependencies so parallel builds work.
@ -139,6 +136,16 @@ if ENABLE_CDF5
TESTS += test_keywords.sh
endif
if ! ISMINGW
if ! ISCYGWIN
TESTS += tst_output.sh tst_nccopy3.sh
TESTS += test_unicode_directory.sh test_unicode_path.sh
if USE_HDF5
TESTS += run_back_comp_tests.sh tst_netcdf4_4.sh
endif
endif
endif
endif BUILD_TESTSETS
# These files all have to be included with the distribution.
@ -228,4 +235,5 @@ scope_*.nc copy_scope_*.cdl
# Remove directories
clean-local:
rm -fr rcmergedir
rm -fr rcmergedir rchome

View File

@ -1,6 +1,6 @@
|ncrc_home|->|ncrc|
|daprc_home|->|daprc|
|dodsrc_home|->|dodsrc|
|ncrc_local|->|ncrc|
|daprc_local|->|daprc|
|dodsrc_home|->|dodsrc|
|dodsrc_local|->|dodsrc|
|ncrc_home|->|ncrc|
|ncrc_local|->|ncrc|

View File

@ -1,3 +1,3 @@
|ncrc|->|ncrc|
|daprc|->|daprc|
|dodsrc|->|dodsrc|
|ncrc|->|ncrc|

View File

@ -1,3 +1,3 @@
|daprc|->|daprc|
|ncrc|->|ncrc2|
|ncrcx|->|ncrcy|
|daprc|->|daprc|

View File

@ -8,13 +8,14 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
#
# 1. Use NCRCENV_RC environment variable exclusively if defined
# 2. If NCRCENV_RC is not defined then merge the set of rc files in this order:
# 1. $HOME/.ncrc
# 2. $HOME/.daprc
# 3. $HOME/.docsrc
# 1. $RCHOME/.ncrc
# 2. $RCHOME/.daprc
# 3. $RCHOME/.docsrc
# 4. $CWD/.ncrc
# 5. $CWD/.daprc
# 6. $CWD/.docsrc
# Entries in later files override any of the >earlier >files
# RCHOME overrides HOME when searching for .rc files.
# Since this involves a shared resource: the .rc files in current working directory,
# we need to isolate from any other test.
@ -24,6 +25,21 @@ WD=`pwd`
cd $srcdir ; abs_srcdir=`pwd` ; cd $WD
cd $execdir ; abs_execdir=`pwd` ; cd $WD
#DEBUG=1
#TRUEHOME=1
# Create RCHOME
if test "x$TRUEHOME" = x1 ; then
RCHOME="$HOME"
else
rm -fr rchome
mkdir rchome
cd rchome
RCHOME=`pwd`
cd ..
export NCRCENV_HOME="$RCHOME"
fi
# Now create a special directory
# And enter it to execute tests
rm -fr rcmergedir
@ -31,36 +47,40 @@ mkdir rcmergedir
cd rcmergedir
WD=`pwd`
if test "x$NCAUTH_HOMETEST" != x ; then
RCHOME=1
fi
HOMERCFILES="$HOME/.ncrc $HOME/.daprc $HOME/.dodsrc"
HOMERCFILES="$RCHOME/.ncrc $RCHOME/.daprc $RCHOME/.dodsrc"
LOCALRCFILES="$WD/.ncrc $WD/.daprc $WD/.dodsrc"
resetrc() {
if test "x$RCHOME" = x1 ; then
rm -f $HOMERCFILES
fi
rm -fr $HOMERCFILES
rm -f $LOCALRCFILES
unset NCRCENV_RC
rm -f tmpoutput.txt
rm -f allfiles1 allfiles2 allfiles3
}
union() {
if test "x$DEBUG" = x1 ; then
rm -f ../allfiles$1
for f in $HOMERCFILES $LOCALRCFILES; do
if test -f $f ; then cat $f >> ../allfiles$1 ; fi
done
fi
}
mergecase1() {
# create everything with different keys to test merge
resetrc
rm -f tmp_rcmerge.txt tmpoutput.txt
for r in "ncrc" "daprc" "dodsrc" ; do
if test "x$RCHOME" = x1 ; then echo "${r}_home=${r}" >> $HOME/".${r}"; fi
echo "for r=ncrc daprc dodsrc"
for r in "ncrc" "daprc" "dodsrc"; do
echo "${r}_home=${r}" >> $RCHOME/".${r}";
echo "${r}_local=${r}" >> $WD/".${r}"
done;
${abs_execdir}/tst_rcmerge > tmpoutput.txt
if test "x$RCHOME" = x1 ; then
cp ${abs_srcdir}/ref_rcmerge1.txt tmp_rcmerge1.txt
else
sed -e '/_local/p' -e d <${abs_srcdir}/ref_rcmerge1.txt > tmp_rcmerge1.txt
fi
diff -b tmp_rcmerge1.txt tmpoutput.txt
union 1
${abs_execdir}/tst_rcmerge |sort > tmpoutput.txt
# echo ">>merge1"; cat ${abs_srcdir}/ref_rcmerge1.txt;
# echo "====="; cat tmpoutput.txt
diff -b ${abs_srcdir}/ref_rcmerge1.txt tmpoutput.txt
}
mergecase2() {
@ -68,10 +88,11 @@ mergecase2() {
resetrc
rm -f tmp_rcmerge.txt tmpoutput.txt
for r in "ncrc" "daprc" "dodsrc" ; do
if test "x$RCHOME" = x1 ; then echo "${r}=${r}" >> $HOME/".${r}"; fi
echo "${r}=${r}" >> $RCHOME/".${r}";
echo "${r}=${r}" >> $WD/".${r}"
done;
${abs_execdir}/tst_rcmerge > tmpoutput.txt
union 2
${abs_execdir}/tst_rcmerge |sort > tmpoutput.txt
diff -b ${abs_srcdir}/ref_rcmerge2.txt tmpoutput.txt
}
@ -79,20 +100,18 @@ mergecase3() {
# Test cross file overrides
resetrc
rm -f tmp_rcmerge.txt tmpoutput.txt
if test "x$RCHOME" = x1 ; then
echo "ncrc=ncrc1" >> $HOME/.ncrc
echo "ncrcx=ncrcx" >> $HOME/.ncrc
echo "ncrc=ncrc2" >> $HOME/.dodsrc
echo "daprc=daprc" >> $HOME/.daprc
else
echo "ncrc=ncrc1" >> $WD/.ncrc
echo "ncrcx=ncrcx" >> $WD/.ncrc
echo "ncrc=ncrc2" >> $WD/.dodsrc
echo "daprc=daprc" >> $WD/.daprc
fi
echo "ncrc=ncrc1" >> $HOME/.ncrc
echo "ncrcx=ncrcx" >> $RCHOME/.ncrc
echo "ncrc=ncrc2" >> $RCHOME/.dodsrc
echo "daprc=daprc" >> $RCHOME/.daprc
echo "ncrc=ncrc1" >> $WD/.ncrc
echo "ncrcx=ncrcx" >> $WD/.ncrc
echo "ncrc=ncrc2" >> $WD/.dodsrc
echo "daprc=daprc" >> $WD/.daprc
echo "daprc=daprc" >> $WD/.dodsrc
echo "ncrcx=ncrcy" >> $WD/.dodsrc
${abs_execdir}/tst_rcmerge > tmpoutput.txt
union 3
${abs_execdir}/tst_rcmerge |sort -d > tmpoutput.txt
diff -b ${abs_srcdir}/ref_rcmerge3.txt tmpoutput.txt
}

View File

@ -671,7 +671,6 @@ Return the value.
static unsigned long long
parseULL(int radix, char* text, int* failp)
{
extern int errno;
char* endptr;
unsigned long long uint64 = 0;

View File

@ -11,6 +11,7 @@ TOPBUILDDIR='@abs_top_builddir@'
FP_ISCMAKE=@ISCMAKE@
FP_ISMSVC=@ISMSVC@
FP_ISCYGWIN=@ISCYGWIN@
FP_ISMINGW=@ISMINGW@
# Feature flags
FEATURE_HDF5=@HAS_HDF5@
@ -75,6 +76,13 @@ set -e
# Allow global set -x mechanism for debugging.
if test "x$SETX" = x1 ; then set -x ; fi
# On MINGW, bash and other POSIX utilities use a mounted root directory,
# but executables compiled for Windows do not recognise the mount point.
# Here we ensure that Windows paths are used in tests of Windows executables.
if test "x@ISMINGW@" = xyes; then
alias pwd='pwd -W'
fi
# We assume that TOPSRCDIR and TOPBUILDDIR are defined
# At the top of this shell script
top_srcdir="$TOPSRCDIR"

View File

@ -93,7 +93,7 @@ generatestrings(int n, unsigned seed)
len = rnd % MAXSTRLEN;
/* generate the characters */
for(k=0;k<len;k++) {
do {rnd = random() % 127;} while(rnd < ' ');
do {rnd = random() % 127;} while(rnd <= ' ');
assert(rnd > ' ' && rnd < 127);
s[k] = (char)rnd;
}