From b3ec7140e19f3505c115488ac57c95b203934537 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 14 Jul 2020 19:24:20 -0600 Subject: [PATCH] Move closer to getting S3 support work with CMake under Visual Studio. The code will build and all the tests will run except for the S3 tests in nczarr_test. --- CMakeLists.txt | 2 +- NUG/nczarr.md | 4 ++-- config.h.cmake.in | 6 ++++++ libnczarr/zs3sdk.cpp | 5 ++++- nczarr_test/CMakeLists.txt | 15 ++++++--------- nczarr_test/run_it_test1.sh | 2 ++ nczarr_test/run_it_test2.sh | 2 ++ nczarr_test/run_ut_mapapi.sh | 6 ++++-- nczarr_test/ut_mapapi.c | 4 ++-- nczarr_test/ut_test.c | 16 ++++++++-------- nczarr_test/ut_util.c | 30 +++++++++++++++--------------- 11 files changed, 52 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e5bc4f24..8eefb0184 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1740,7 +1740,7 @@ ENDIF(ENABLE_BASH_SCRIPT_TESTING) MACRO(add_sh_test prefix F) IF(HAVE_BASH) - ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh") + ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${F}.sh ${ARGN}") ENDIF() ENDMACRO() diff --git a/NUG/nczarr.md b/NUG/nczarr.md index 401564419..296306034 100755 --- a/NUG/nczarr.md +++ b/NUG/nczarr.md @@ -341,10 +341,10 @@ Operating System | Build System | NCZarr | S3 Support ----------------------------------------------------------- Linux | Automake | yes | yes Linux | CMake | yes | yes -Cygwin | Automake | yes | unknown +Cygwin | Automake | yes | no OSX | Automake | unknown | unknown OSX | CMake | unknown | unknown -Visual Studio | CMake | yes | unknown +Visual Studio | CMake | yes | tests fail Note: S3 support includes both compiling the S3 support code as well as running the S3 tests. diff --git a/config.h.cmake.in b/config.h.cmake.in index d50c7ba13..70ef3be24 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -145,6 +145,12 @@ are set when opening a binary file on Windows. */ /* if true, enable NCZARR */ #cmakedefine ENABLE_NCZARR 1 +/* if true, enable S3 support */ +#cmakedefine ENABLE_S3_SDK 1 + +/* if true, enable S3 testing*/ +#cmakedefine ENABLE_S3_TESTS 1 + /* define the possible sources for remote test servers */ #cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" diff --git a/libnczarr/zs3sdk.cpp b/libnczarr/zs3sdk.cpp index 13c6d57bc..df3d395b0 100644 --- a/libnczarr/zs3sdk.cpp +++ b/libnczarr/zs3sdk.cpp @@ -5,7 +5,6 @@ #define NOOP - #include "awsincludes.h" #include #include @@ -14,6 +13,10 @@ #include "netcdf.h" #include "zs3sdk.h" +#ifdef __CYGWIN__ +extern char* strdup(const char*); +#endif + #define size64_t unsigned long long static Aws::SDKOptions zs3options; diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index b76fb718f..7ceccd09e 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -53,15 +53,12 @@ IF(ENABLE_TESTS) TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr) IF(BUILD_UTILITIES) - if(ENABLE_S3_TESTS) - set(ENV{ENABLE_S3_TESTS} "1") - endif() - add_sh_test(nczarr_test run_ut_map) - add_sh_test(nczarr_test run_ut_mapapi) - add_sh_test(nczarr_test run_ut_misc) - add_sh_test(nczarr_test run_ut_chunk) - add_sh_test(nczarr_test run_it_test1) - add_sh_test(nczarr_test run_it_test2) + add_sh_test(nczarr_test run_ut_map ${ENABLE_S3_TESTS}) + add_sh_test(nczarr_test run_ut_mapapi ${ENABLE_S3_TESTS}) + add_sh_test(nczarr_test run_ut_misc ${ENABLE_S3_TESTS}) + add_sh_test(nczarr_test run_ut_chunk ${ENABLE_S3_TESTS}) + add_sh_test(nczarr_test run_it_test1 ${ENABLE_S3_TESTS}) + add_sh_test(nczarr_test run_it_test2 ${ENABLE_S3_TESTS}) ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_TESTS) diff --git a/nczarr_test/run_it_test1.sh b/nczarr_test/run_it_test1.sh index 7c1784251..6dbddb870 100755 --- a/nczarr_test/run_it_test1.sh +++ b/nczarr_test/run_it_test1.sh @@ -1,5 +1,7 @@ #!/bin/sh +ENABLE_S3_TESTS=$1 + if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/nczarr_test/run_it_test2.sh b/nczarr_test/run_it_test2.sh index 06910753e..9a4971e07 100755 --- a/nczarr_test/run_it_test2.sh +++ b/nczarr_test/run_it_test2.sh @@ -1,5 +1,7 @@ #!/bin/sh +ENABLE_S3_TESTS=$1 + if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/nczarr_test/run_ut_mapapi.sh b/nczarr_test/run_ut_mapapi.sh index 1fa319304..f41b4b10d 100755 --- a/nczarr_test/run_ut_mapapi.sh +++ b/nczarr_test/run_ut_mapapi.sh @@ -1,5 +1,7 @@ #!/bin/sh +ENABLE_S3_TESTS=$1 + if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh @@ -80,8 +82,8 @@ testmapcreate nz4; testmapmeta nz4; testmapdata nz4; testmapsearch nz4 echo ""; echo "*** Test zmap_nzf" testmapcreate nzf; testmapmeta nzf; testmapdata nzf; testmapsearch nzf if test "x$ENABLE_S3_TESTS" != x ; then -echo ""; echo "*** Test zmap_s3sdk" -testmapcreate s3; testmapmeta s3; testmapdata s3; testmapsearch s3 + echo ""; echo "*** Test zmap_s3sdk" + testmapcreate s3; testmapmeta s3; testmapdata s3; testmapsearch s3 fi } diff --git a/nczarr_test/ut_mapapi.c b/nczarr_test/ut_mapapi.c index 09a82fd77..d450423bf 100644 --- a/nczarr_test/ut_mapapi.c +++ b/nczarr_test/ut_mapapi.c @@ -62,7 +62,7 @@ main(int argc, char** argv) done: nullfree(url); url = NULL; nullfree(keyprefix); - if(stat) usage(stat); + if(stat) usage(THROW(stat)); return 0; } @@ -356,7 +356,7 @@ searchR(NCZMAP* map, int depth, const char* prefix, NClist* objects) done: nclistfreeall(matches); - return stat; + return THROW(stat); } static int diff --git a/nczarr_test/ut_test.c b/nczarr_test/ut_test.c index dbf1e5e60..93fe01127 100755 --- a/nczarr_test/ut_test.c +++ b/nczarr_test/ut_test.c @@ -59,7 +59,7 @@ ut_init(int argc, char** argv, struct UTOptions * options) options->debug = 1; break; case 'x': /*execute*/ - if(parsestringvector(optarg,0,&options->cmds) <= 0) usage(0); + if(parsestringvector(optarg,0,&options->cmds) <= 0) usage(THROW(0)); break; case 'f': options->file = strdup(optarg); @@ -71,17 +71,17 @@ ut_init(int argc, char** argv, struct UTOptions * options) options->kind = strdup(optarg); break; case 'd': /*dimdef*/ - if((stat=parsedimdef(optarg,&dimdef))) usage(stat); + if((stat=parsedimdef(optarg,&dimdef))) usage(THROW(stat)); nclistpush(options->dimdefs,dimdef); dimdef = NULL; break; case 'v': /*vardef*/ - if((stat=parsevardef(optarg,options->dimdefs,&vardef))) usage(stat); + if((stat=parsevardef(optarg,options->dimdefs,&vardef))) usage(THROW(stat)); nclistpush(options->vardefs,vardef); vardef = NULL; break; case 's': /*slices*/ - if((stat=parseslices(optarg,&options->nslices,options->slices))) usage(stat); + if((stat=parseslices(optarg,&options->nslices,options->slices))) usage(THROW(stat)); break; case 'W': /*walk data*/ options->idatalen = parseintvector(optarg,4,(void**)&options->idata); @@ -98,7 +98,7 @@ ut_init(int argc, char** argv, struct UTOptions * options) canonicalfile(&options->output); done: - return stat; + return THROW(stat); } static void @@ -233,15 +233,15 @@ runtests(const char** cmds, struct Test* tests) int stat = NC_NOERR; struct Test* test = NULL; const char** cmd = NULL; - if(cmds == NULL) return NC_EINVAL; + if(cmds == NULL) return THROW(NC_EINVAL); for(cmd=cmds;*cmd;cmd++) { for(test=tests;test->cmd;test++) { if(strcmp(test->cmd,*cmd)==0) { - if(test->cmd == NULL) return NC_EINVAL; + if(test->cmd == NULL) return THROW(NC_EINVAL); if((stat=test->test())) goto done; /* Execute */ } } } done: - return stat; + return THROW(stat); } diff --git a/nczarr_test/ut_util.c b/nczarr_test/ut_util.c index fb9562878..1bc6efed8 100644 --- a/nczarr_test/ut_util.c +++ b/nczarr_test/ut_util.c @@ -42,7 +42,7 @@ parseslices(const char* s0, int* nslicesp, NCZSlice* slices) if(*s == '[') nslices++; } - if(nslices > NC_MAX_VAR_DIMS) return NC_EINVAL; /* too many */ + if(nslices > NC_MAX_VAR_DIMS) return THROW(NC_EINVAL); /* too many */ if(nslicesp) *nslicesp = nslices; /* Extract the slices */ @@ -55,7 +55,7 @@ parseslices(const char* s0, int* nslicesp, NCZSlice* slices) if(nchars == -1) { nchars = -1; count = sscanf(s,"[%lu:%lu:%lu]%n",&start,&stop,&stride,&nchars); - if(count != 3) return NC_EINVAL; + if(count != 3) return THROW(NC_EINVAL); } sl->start = start; sl->stop = stop; @@ -75,12 +75,12 @@ parsedimdef(const char* s0, Dimdef** defp) ptrdiff_t count; if((def = calloc(1,sizeof(struct Dimdef)))==NULL) - return NC_ENOMEM; + return THROW(NC_ENOMEM); /* Extract */ s = s0; if((p = strchr(s,'=')) == NULL) abort(); - if((count = (p - s)) == 0) return NC_EINVAL; + if((count = (p - s)) == 0) return THROW(NC_EINVAL); def->name = malloc(count+1); memcpy(def->name,s,count); def->name[count] = '\0'; @@ -105,15 +105,15 @@ parsevardef(const char* s0, NClist* dimdefs, Vardef** varp) char name[NC_MAX_NAME]; if((vd = calloc(1,sizeof(Vardef)))==NULL) - return NC_ENOMEM; + return THROW(NC_ENOMEM); s=s0; /* Scan for the end of type name */ p = strchr(s,BLANK); - if(p == NULL) return NC_EINVAL; + if(p == NULL) return THROW(NC_EINVAL); len = (p - s); - if(len == 0) return NC_EINVAL; + if(len == 0) return THROW(NC_EINVAL); memcpy(name,s,len); name[len] = '\0'; vd->typeid = ut_typeforname(name); @@ -123,9 +123,9 @@ parsevardef(const char* s0, NClist* dimdefs, Vardef** varp) /* Scan for the end of var name */ p = strchr(s,LPAREN); - if(p == NULL) return NC_EINVAL; + if(p == NULL) return THROW(NC_EINVAL); len = (p - s); - if(len == 0) return NC_EINVAL; + if(len == 0) return THROW(NC_EINVAL); memcpy(name,s,len); name[len] = '\0'; vd->name = strdup(name); @@ -136,7 +136,7 @@ parsevardef(const char* s0, NClist* dimdefs, Vardef** varp) char* p; s++; count = parsestringvector(s,RPAREN,&names); - if(count >= NC_MAX_VAR_DIMS) return NC_EINVAL; + if(count >= NC_MAX_VAR_DIMS) return THROW(NC_EINVAL); vd->rank = count; if(vd->rank > 0) { int j; @@ -146,7 +146,7 @@ parsevardef(const char* s0, NClist* dimdefs, Vardef** varp) p = strchr(names[j],'/'); if(p) *p++ = '\0'; if((dimref = finddim(names[j],dimdefs)) == NULL) - return NC_EINVAL; + return THROW(NC_EINVAL); vd->dimrefs[j] = dimref; vd->dimsizes[j] = dimref->size; if(p == NULL) @@ -176,7 +176,7 @@ parsestringvector(const char* s0, int stopchar, char*** namesp) /* First, compute number of elements */ for(s=s0,nelems=1;*s;s++) {if(*s == ',') nelems++; if(*s == stopchar) break;} - if(nelems == 0) return NC_EINVAL; + if(nelems == 0) return THROW(NC_EINVAL); names = calloc(nelems+1,sizeof(char*)); for(s=s0,i=0;i