mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-04-06 18:00:24 +08:00
Upgrade the nczarr code to match Zarr V2
Re: https://github.com/zarr-developers/zarr-python/pull/716 The Zarr version 2 spec has been extended to include the ability to choose the dimension separator in chunk name keys. The legal separators has been extended from {'.'} to {'.' '/'}. So now it is possible to use a key like "0/1/2/0" for chunk names. This PR implements this for NCZarr. The V2 spec now says that this separator can be set on a per-variable basis. For now, I have chosen to allow this be set only globally by adding a key named "ZARR.DIMENSION_SEPARATOR=<char>" in the .daprc/.dodsrc/ncrc file. Currently, the only legal separator characters are '.' (the default) and '/'. On writing, this key will only be written if its value is different than the default. This change caused problems because supporting a separator of '/' is difficult to parse when keys/paths use '/' as the path separator. A test case was added for this. Additionally, make nczarr be enabled default by default. This required some additional changes so that if zip and/or AWS S3 sdk are unavailable, then they are disabled for NCZarr. In addition the following unrelated changes were made. 1. Tested that pure-zarr mode could read an nczarr formatted store. 1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added. 1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl is not found then the other options that depend upon it properly are disabled. 1. I decided that xarray support should be enabled by default for pure zarr. In order to allow disabling, I added a new mode flag "noxarray". 1. Certain test in nczarr_test depend on use of .dodsrc. In order for these to work when testing in parallel, some inter-test dependencies needed to be added. 1. Improved authorization testing to use changes in thredds.ucar.edu
This commit is contained in:
Dennis Heimbigner
parent
706e33b056
commit
74b40fd788
2
.github/workflows/run_tests.yml
vendored
2
.github/workflows/run_tests.yml
vendored
@ -4,7 +4,7 @@
|
||||
|
||||
name: Run netCDF Tests
|
||||
|
||||
on: [pull_request]
|
||||
on: [pull_request,push]
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
@ -1071,11 +1071,14 @@ IF(NOT FOUND_CURL)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_DAP2 OR ENABLE_DAP4)
|
||||
MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.")
|
||||
MESSAGE(WARNING "CURL libraries are not found; DAP support disabled")
|
||||
SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE)
|
||||
SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_HDF5_ROS3)
|
||||
MESSAGE(FATAL_ERROR "ROS3 support specified, CURL libraries are not found.")
|
||||
MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.")
|
||||
SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
|
||||
ENDIF()
|
||||
|
||||
IF(ENABLE_NCZARR_S3)
|
||||
@ -2185,7 +2188,7 @@ is_enabled(STATUS_PNETCDF HAS_PNETCDF)
|
||||
is_enabled(STATUS_PARALLEL HAS_PARALLEL)
|
||||
is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
|
||||
is_enabled(ENABLE_DAP HAS_DAP)
|
||||
is_enabled(ENABLE_DAP HAS_DAP2)
|
||||
is_enabled(ENABLE_DAP2 HAS_DAP2)
|
||||
is_enabled(ENABLE_DAP4 HAS_DAP4)
|
||||
is_enabled(ENABLE_BYTERANGE HAS_BYTERANGE)
|
||||
is_enabled(ENABLE_DISKLESS HAS_DISKLESS)
|
||||
|
||||
126
configure.ac
126
configure.ac
@ -122,6 +122,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
|
||||
test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
|
||||
AC_MSG_RESULT($enable_pnetcdf)
|
||||
|
||||
# We need curl for remote operations
|
||||
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
|
||||
if test "x$found_curl" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],[])
|
||||
fi
|
||||
|
||||
## Capture the state of the --enable-dap flag => enable dap2+dap4
|
||||
AC_MSG_CHECKING([whether DAP client(s) are to be built])
|
||||
AC_ARG_ENABLE([dap],
|
||||
@ -130,11 +136,16 @@ AC_ARG_ENABLE([dap],
|
||||
test "x$enable_dap" = xno || enable_dap=yes
|
||||
AC_MSG_RESULT($enable_dap)
|
||||
|
||||
AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
|
||||
if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
|
||||
AC_MSG_WARN([curl required for dap access. DAP support disabled.])
|
||||
enable_dap=no
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([whether netcdf zarr storage format should be disabled])
|
||||
AC_ARG_ENABLE([nczarr],
|
||||
[AS_HELP_STRING([--enable-nczarr],
|
||||
[enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
|
||||
test "x$enable_nczarr" = xyes || enable_nczarr=no
|
||||
[AS_HELP_STRING([--disable-nczarr],
|
||||
[disable netcdf zarr storage support])])
|
||||
test "x$enable_nczarr" = xno || enable_nczarr=yes
|
||||
AC_MSG_RESULT($enable_nczarr)
|
||||
|
||||
# HDF5 | HDF4 | NCZarr => netcdf-4
|
||||
@ -395,12 +406,6 @@ if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then
|
||||
fi
|
||||
AC_MSG_RESULT($enable_set_log_level_func)
|
||||
|
||||
# We need curl for DAP and byterange
|
||||
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
|
||||
if test "x$found_curl" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],
|
||||
[AC_MSG_ERROR([curl required for remote access. Install curl or disable relevant options.])])
|
||||
fi
|
||||
|
||||
# CURLOPT_USERNAME is not defined until curl version 7.19.1
|
||||
# CURLOPT_PASSWORD is not defined until curl version 7.19.1
|
||||
@ -560,25 +565,74 @@ AC_MSG_RESULT([$enable_dap_long_tests])
|
||||
|
||||
# Control zarr storage
|
||||
if test "x$enable_nczarr" = xyes ; then
|
||||
if test "x$enable_netcdf_4" = xno ; then
|
||||
AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr])
|
||||
enable_nczarr=no
|
||||
fi
|
||||
if test "x$enable_netcdf_4" = xno ; then
|
||||
AC_MSG_WARN([netCDF-4 disabled, so you must not enable nczarr])
|
||||
enable_nczarr=no
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "x$enable_nczarr" = xyes; then
|
||||
AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client])
|
||||
AC_SUBST(ENABLE_NCZARR)
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
|
||||
|
||||
# See if we have libzip
|
||||
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
|
||||
if test "x$have_zip" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
|
||||
fi
|
||||
AC_MSG_CHECKING([whether libzip library is available])
|
||||
AC_MSG_RESULT([${have_zip}])
|
||||
|
||||
enable_nczarr_zip=${have_zip} # alias
|
||||
|
||||
if test "x$enable_nczarr" = xno ; then
|
||||
enable_nczarr_zip=no
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([whether nczarr zip support is enabled])
|
||||
AC_MSG_RESULT([${enable_nczarr_zip}])
|
||||
|
||||
if test "x$enable_nczarr_zip" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
|
||||
|
||||
# Check for enabling of S3 support
|
||||
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
|
||||
AC_ARG_ENABLE([nczarr-s3],
|
||||
[AS_HELP_STRING([--enable-nczarr-s3],
|
||||
[enable netcdf zarr S3 support; make sure to set LDFLAGS])])
|
||||
test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
|
||||
if test "x$enable_nczarr" = xno ; then
|
||||
enable_nczarr_s3=no
|
||||
fi
|
||||
AC_MSG_RESULT($enable_nczarr_s3)
|
||||
|
||||
# Note we check for the library after checking for enable_nczarr_s3
|
||||
# because for some reason this screws up if we unconditionally test for sdk
|
||||
# and it is not available. Fix someday
|
||||
have_aws=no
|
||||
if test "x$enable_nczarr_s3" = xyes ; then
|
||||
# See if we have the s3 aws library
|
||||
# Check for the AWS S3 SDK library
|
||||
AC_LANG_PUSH([C++])
|
||||
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
|
||||
AC_LANG_POP
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
|
||||
AC_MSG_RESULT([$have_aws])
|
||||
|
||||
if test "x$have_aws" = xno ; then
|
||||
AC_MSG_WARN([AWS SDK not found; disabling S3 support])
|
||||
enable_nczarr_s3=no
|
||||
else
|
||||
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
|
||||
|
||||
# Check for enabling S3 testing
|
||||
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
|
||||
AC_ARG_ENABLE([nczarr-s3-tests],
|
||||
@ -589,36 +643,10 @@ AC_MSG_RESULT($enable_nczarr_s3_tests)
|
||||
|
||||
# Disable S3 tests if S3 support is disabled
|
||||
if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
|
||||
AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests])
|
||||
enable_nczarr_s3_tests=no
|
||||
fi
|
||||
|
||||
# Set default
|
||||
have_aws=no
|
||||
|
||||
if test "x$enable_nczarr_s3" = xyes ; then
|
||||
# See if we have the s3 aws library
|
||||
# Check for the AWS S3 SDK library
|
||||
AC_LANG_PUSH([C++])
|
||||
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
|
||||
AC_LANG_POP
|
||||
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
|
||||
AC_MSG_RESULT([$have_aws])
|
||||
|
||||
if test "x$have_aws" = xyes ; then
|
||||
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
|
||||
fi
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
|
||||
|
||||
if test "x$have_aws" = xno ; then
|
||||
if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
|
||||
enable_nczarr_s3_tests=no
|
||||
enable_nczarr_s3=no
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "x$enable_nczarr_s3" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
|
||||
fi
|
||||
@ -634,22 +662,6 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
|
||||
fi
|
||||
|
||||
# Set default
|
||||
# See if we have libzip
|
||||
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
|
||||
if test "x$have_zip" = "xyes" ; then
|
||||
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [],
|
||||
[AC_MSG_ERROR([libzip search failed.])])
|
||||
fi
|
||||
AC_MSG_CHECKING([whether libzip library is available])
|
||||
AC_MSG_RESULT([${have_zip}])
|
||||
|
||||
enable_nczarr_zip=${have_zip} # alias
|
||||
|
||||
if test "x$enable_nczarr_zip" = xyes ; then
|
||||
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
|
||||
fi
|
||||
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
|
||||
|
||||
# Did the user specify a default cache size for NCZarr?
|
||||
AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
|
||||
AC_ARG_WITH([chunk-cache-size-nczarr],
|
||||
|
||||
@ -117,8 +117,7 @@ EXTERNL int NCaccess(const char* path, int mode);
|
||||
EXTERNL int NCremove(const char* path);
|
||||
EXTERNL int NCmkdir(const char* path, int mode);
|
||||
EXTERNL int NCrmdir(const char* path);
|
||||
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
|
||||
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
|
||||
EXTERNL char* NCgetcwd(char* cwdbuf, size_t len);
|
||||
#ifdef HAVE_SYS_STAT_H
|
||||
EXTERNL int NCstat(char* path, struct stat* buf);
|
||||
#endif
|
||||
|
||||
@ -16,6 +16,11 @@ and accessing rc files (e.g. .daprc).
|
||||
#include "nclist.h"
|
||||
#include "ncbytes.h"
|
||||
|
||||
/* getenv() keys */
|
||||
#define NCRCENVIGNORE "NCRCENV_IGNORE"
|
||||
#define NCRCENVRC "NCRCENV_RC"
|
||||
|
||||
|
||||
typedef struct NCTriple {
|
||||
char* host; /* combined host:port */
|
||||
char* key;
|
||||
@ -34,31 +39,40 @@ typedef struct NCRCinfo {
|
||||
typedef struct NCRCglobalstate {
|
||||
int initialized;
|
||||
char* tempdir; /* track a usable temp dir */
|
||||
char* home; /* track $HOME for use in creating $HOME/.oc dir */
|
||||
char* home; /* track $HOME */
|
||||
char* cwd; /* track getcwd */
|
||||
NCRCinfo rcinfo; /* Currently only one rc file per session */
|
||||
struct GlobalZarr { /* Zarr specific parameters */
|
||||
char dimension_separator;
|
||||
} zarr;
|
||||
} NCRCglobalstate;
|
||||
|
||||
/* From drc.c */
|
||||
extern NCRCglobalstate* ncrc_getglobalstate(void);
|
||||
extern void ncrc_freeglobalstate(void);
|
||||
EXTERNL void ncrc_initialize(void);
|
||||
EXTERNL void ncrc_freeglobalstate(void);
|
||||
/* read and compile the rc file, if any */
|
||||
extern int NC_rcload(void);
|
||||
extern char* NC_rclookup(const char* key, const char* hostport);
|
||||
extern void NC_rcclear(NCRCinfo* info);
|
||||
extern int NC_set_rcfile(const char* rcfile);
|
||||
extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
|
||||
EXTERNL int NC_rcload(void);
|
||||
EXTERNL char* NC_rclookup(const char* key, const char* hostport);
|
||||
EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
|
||||
|
||||
/* Following are primarily for debugging */
|
||||
/* Obtain the count of number of triples */
|
||||
extern size_t NC_rcfile_length(NCRCinfo*);
|
||||
EXTERNL size_t NC_rcfile_length(NCRCinfo*);
|
||||
/* Obtain the ith triple; return NULL if out of range */
|
||||
extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
|
||||
EXTERNL NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
|
||||
|
||||
/* For internal use */
|
||||
EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
|
||||
EXTERNL void NC_rcclear(NCRCinfo* info);
|
||||
EXTERNL void NC_rcclear(NCRCinfo* info);
|
||||
|
||||
/* From dutil.c (Might later move to e.g. nc.h */
|
||||
extern int NC__testurl(const char* path, char** basenamep);
|
||||
extern int NC_isLittleEndian(void);
|
||||
extern char* NC_entityescape(const char* s);
|
||||
extern int NC_readfile(const char* filename, NCbytes* content);
|
||||
extern int NC_writefile(const char* filename, size_t size, void* content);
|
||||
extern char* NC_mktmp(const char* base);
|
||||
extern int NC_getmodelist(const char* url, NClist** modelistp);
|
||||
extern int NC_testmode(const char* path, const char* tag);
|
||||
EXTERNL int NC__testurl(const char* path, char** basenamep);
|
||||
EXTERNL int NC_isLittleEndian(void);
|
||||
EXTERNL char* NC_entityescape(const char* s);
|
||||
EXTERNL int NC_readfile(const char* filename, NCbytes* content);
|
||||
EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
|
||||
EXTERNL char* NC_mktmp(const char* base);
|
||||
EXTERNL int NC_getmodelist(const char* url, NClist** modelistp);
|
||||
EXTERNL int NC_testmode(const char* path, const char* tag);
|
||||
#endif /*NCRC_H*/
|
||||
|
||||
@ -32,8 +32,6 @@ size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0};
|
||||
size_t NC_coord_one[NC_MAX_VAR_DIMS] = {1};
|
||||
ptrdiff_t NC_stride_one[NC_MAX_VAR_DIMS] = {1};
|
||||
|
||||
NCRCglobalstate ncrc_globalstate;
|
||||
|
||||
/*
|
||||
static nc_type longtype = (sizeof(long) == sizeof(int)?NC_INT:NC_INT64);
|
||||
static nc_type ulongtype = (sizeof(unsigned long) == sizeof(unsigned int)?NC_UINT:NC_UINT64);
|
||||
@ -81,6 +79,20 @@ NCDISPATCH_initialize(void)
|
||||
globalstate->home = strdup(home);
|
||||
}
|
||||
|
||||
/* Capture $CWD */
|
||||
{
|
||||
char cwdbuf[4096];
|
||||
|
||||
cwdbuf[0] = '\0';
|
||||
(void)NCgetcwd(cwdbuf,sizeof(cwdbuf));
|
||||
|
||||
if(strlen(cwdbuf) == 0) {
|
||||
/* use tempdir */
|
||||
strcpy(cwdbuf, globalstate->tempdir);
|
||||
}
|
||||
globalstate->cwd = strdup(cwdbuf);
|
||||
}
|
||||
|
||||
/* Now load RC File */
|
||||
status = NC_rcload();
|
||||
ncloginit();
|
||||
|
||||
@ -123,6 +123,7 @@ static const struct MACRODEF {
|
||||
{"s3","mode","nczarr,s3"},
|
||||
{"bytes","mode","bytes"},
|
||||
{"xarray","mode","nczarr,zarr,xarray"},
|
||||
{"noxarray","mode","nczarr,zarr,noxarray"},
|
||||
{NULL,NULL,NULL}
|
||||
};
|
||||
|
||||
@ -133,6 +134,7 @@ static const struct MODEINFER {
|
||||
} modeinferences[] = {
|
||||
{"zarr","nczarr"},
|
||||
{"xarray","zarr"},
|
||||
{"noxarray","zarr"},
|
||||
{NULL,NULL}
|
||||
};
|
||||
|
||||
|
||||
@ -20,8 +20,6 @@ See COPYRIGHT for license information.
|
||||
#include "nclog.h"
|
||||
#include "ncpathmgr.h"
|
||||
|
||||
#define RCFILEENV "DAPRCFILE"
|
||||
|
||||
#define RTAG ']'
|
||||
#define LTAG '['
|
||||
|
||||
@ -42,21 +40,41 @@ static void rcfreetriples(NClist* rc);
|
||||
static void storedump(char* msg, NClist* triples);
|
||||
#endif
|
||||
|
||||
/* Define default rc files and aliases, also defines search order*/
|
||||
static const char* rcfilenames[] = {".daprc",".dodsrc",".ncrc",NULL};
|
||||
/* Define default rc files and aliases, also defines load order*/
|
||||
static const char* rcfilenames[] = {".ncrc", ".daprc", ".dodsrc",NULL};
|
||||
|
||||
/**************************************************/
|
||||
/* External Entry Points */
|
||||
|
||||
static NCRCglobalstate* ncrc_globalstate = NULL;
|
||||
|
||||
static int NCRCinitialized = 0;
|
||||
|
||||
/* Initialize defaults */
|
||||
void
|
||||
ncrc_initialize(void)
|
||||
{
|
||||
const char* tmp = NULL;
|
||||
|
||||
if(NCRCinitialized) return;
|
||||
if(ncrc_globalstate == NULL) {
|
||||
ncrc_globalstate = calloc(1,sizeof(NCRCglobalstate));
|
||||
}
|
||||
/* Get environment variables */
|
||||
if(getenv(NCRCENVIGNORE) != NULL)
|
||||
ncrc_globalstate->rcinfo.ignore = 1;
|
||||
tmp = getenv(NCRCENVRC);
|
||||
if(tmp != NULL && strlen(tmp) > 0)
|
||||
ncrc_globalstate->rcinfo.rcfile = strdup(tmp);
|
||||
NCRCinitialized = 1;
|
||||
}
|
||||
|
||||
/* Get global state */
|
||||
NCRCglobalstate*
|
||||
ncrc_getglobalstate(void)
|
||||
{
|
||||
if(ncrc_globalstate == NULL) {
|
||||
ncrc_globalstate = calloc(1,sizeof(NCRCglobalstate));
|
||||
}
|
||||
if(!NCRCinitialized)
|
||||
ncrc_initialize();
|
||||
return ncrc_globalstate;
|
||||
}
|
||||
|
||||
@ -66,6 +84,7 @@ ncrc_freeglobalstate(void)
|
||||
if(ncrc_globalstate != NULL) {
|
||||
nullfree(ncrc_globalstate->tempdir);
|
||||
nullfree(ncrc_globalstate->home);
|
||||
nullfree(ncrc_globalstate->cwd);
|
||||
NC_rcclear(&ncrc_globalstate->rcinfo);
|
||||
free(ncrc_globalstate);
|
||||
ncrc_globalstate = NULL;
|
||||
@ -94,57 +113,67 @@ rcfreetriples(NClist* rc)
|
||||
nclistfree(rc);
|
||||
}
|
||||
|
||||
/* locate, read and compile the rc file, if any */
|
||||
/* locate, read and compile the rc files, if any */
|
||||
int
|
||||
NC_rcload(void)
|
||||
{
|
||||
int ret = NC_NOERR;
|
||||
int i,ret = NC_NOERR;
|
||||
char* path = NULL;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NCRCglobalstate* globalstate = NULL;
|
||||
NClist* rcfileorder = nclistnew();
|
||||
|
||||
if(!NCRCinitialized) ncrc_initialize();
|
||||
globalstate = ncrc_getglobalstate();
|
||||
|
||||
if(globalstate->rcinfo.ignore) {
|
||||
nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing");
|
||||
return (NC_NOERR);
|
||||
nclog(NCLOGDBG,".rc file loading suppressed");
|
||||
goto done;
|
||||
}
|
||||
if(globalstate->rcinfo.loaded) return (NC_NOERR);
|
||||
if(globalstate->rcinfo.loaded) goto done;
|
||||
|
||||
/* locate the configuration files in the following order:
|
||||
1. specified by NC_set_rcfile
|
||||
2. set by DAPRCFILE env variable
|
||||
3. ./<rcfile> (current directory)
|
||||
4. $HOME/<rcfile>
|
||||
/* locate the configuration files in order of use:
|
||||
1. Specified by NCRCENV_RC environment variable.
|
||||
2. If NCRCENV_RC is not set then merge the set of rc files in this order:
|
||||
1. $HOME/.ncrc
|
||||
2. $HOME/.daprc
|
||||
3. $HOME/.docsrc
|
||||
4. $CWD/.ncrc
|
||||
5. $CWD/.daprc
|
||||
6. $CWD/.docsrc
|
||||
Entries in later files override any of the earlier files
|
||||
*/
|
||||
if(globalstate->rcinfo.rcfile != NULL) { /* always use this */
|
||||
path = strdup(globalstate->rcinfo.rcfile);
|
||||
} else if(getenv(RCFILEENV) != NULL && strlen(getenv(RCFILEENV)) > 0) {
|
||||
path = strdup(getenv(RCFILEENV));
|
||||
nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile));
|
||||
} else {
|
||||
const char** rcname;
|
||||
int found = 0;
|
||||
for(rcname=rcfilenames;!found && *rcname;rcname++) {
|
||||
ret = rcsearch(".",*rcname,&path);
|
||||
if(ret == NC_NOERR && path == NULL) /* try $HOME */
|
||||
ret = rcsearch(globalstate->home,*rcname,&path);
|
||||
if(ret != NC_NOERR)
|
||||
goto done;
|
||||
if(path != NULL)
|
||||
found = 1;
|
||||
const char* dirnames[3];
|
||||
const char** dir;
|
||||
|
||||
dirnames[0] = globalstate->home;
|
||||
dirnames[1] = globalstate->cwd;
|
||||
dirnames[2] = NULL;
|
||||
|
||||
for(dir=dirnames;*dir;dir++) {
|
||||
for(rcname=rcfilenames;*rcname;rcname++) {
|
||||
ret = rcsearch(*dir,*rcname,&path);
|
||||
if(ret == NC_NOERR && path != NULL)
|
||||
nclistpush(rcfileorder,path);
|
||||
path = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(path == NULL) {
|
||||
nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing");
|
||||
} else {
|
||||
#ifdef D4DEBUG
|
||||
fprintf(stderr, "RC file: %s\n", path);
|
||||
#endif
|
||||
if((ret=rccompile(path))) {
|
||||
nclog(NCLOGERR, "Error parsing %s\n",path);
|
||||
for(i=0;i<nclistlength(rcfileorder);i++) {
|
||||
path = (char*)nclistget(rcfileorder,i);
|
||||
if((ret=rccompile(path))) {
|
||||
nclog(NCLOGWARN, "Error parsing %s\n",path);
|
||||
ret = NC_NOERR; /* ignore it */
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
globalstate->rcinfo.loaded = 1; /* even if not exists */
|
||||
nullfree(path);
|
||||
nclistfreeall(rcfileorder);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -155,10 +184,13 @@ done:
|
||||
char*
|
||||
NC_rclookup(const char* key, const char* hostport)
|
||||
{
|
||||
struct NCTriple* triple = rclocate(key,hostport);
|
||||
struct NCTriple* triple = NULL;
|
||||
if(!NCRCinitialized) ncrc_initialize();
|
||||
triple = rclocate(key,hostport);
|
||||
return (triple == NULL ? NULL : triple->value);
|
||||
}
|
||||
|
||||
#if 0
|
||||
/*!
|
||||
Set the absolute path to use for the rc file.
|
||||
WARNING: this MUST be called before any other
|
||||
@ -195,6 +227,7 @@ NC_set_rcfile(const char* rcfile)
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
#endif
|
||||
|
||||
/**************************************************/
|
||||
/* RC processing functions */
|
||||
@ -224,6 +257,8 @@ rctrim(char* text)
|
||||
size_t len = 0;
|
||||
int i;
|
||||
|
||||
if(text == NULL) return;
|
||||
|
||||
/* locate first non-trimchar */
|
||||
for(;*p;p++) {
|
||||
if(strchr(TRIMCHARS,*p) == NULL) break; /* hit non-trim char */
|
||||
@ -276,7 +311,7 @@ rcorder(NClist* rc)
|
||||
nclistfree(tmprc);
|
||||
}
|
||||
|
||||
/* Create a triple store from a file */
|
||||
/* Merge a triple store from a file*/
|
||||
static int
|
||||
rccompile(const char* path)
|
||||
{
|
||||
@ -289,24 +324,23 @@ rccompile(const char* path)
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
|
||||
if((ret=NC_readfile(path,tmp))) {
|
||||
nclog(NCLOGERR, "Could not open configuration file: %s",path);
|
||||
nclog(NCLOGWARN, "Could not open configuration file: %s",path);
|
||||
goto done;
|
||||
}
|
||||
contents = ncbytesextract(tmp);
|
||||
if(contents == NULL) contents = strdup("");
|
||||
/* Either reuse or create new */
|
||||
rc = globalstate->rcinfo.triples;
|
||||
if(rc != NULL)
|
||||
rcfreetriples(rc); /* clear out any old data */
|
||||
else {
|
||||
if(rc == NULL) {
|
||||
rc = nclistnew();
|
||||
globalstate->rcinfo.triples = rc;
|
||||
}
|
||||
nextline = contents;
|
||||
for(;;) {
|
||||
char* line;
|
||||
char* key;
|
||||
char* value;
|
||||
char* key = NULL;
|
||||
char* value = NULL;
|
||||
char* host = NULL;
|
||||
size_t llen;
|
||||
NCTriple* triple;
|
||||
|
||||
@ -315,14 +349,11 @@ rccompile(const char* path)
|
||||
rctrim(line); /* trim leading and trailing blanks */
|
||||
if(line[0] == '#') continue; /* comment */
|
||||
if((llen=strlen(line)) == 0) continue; /* empty line */
|
||||
triple = (NCTriple*)calloc(1,sizeof(NCTriple));
|
||||
if(triple == NULL) {ret = NC_ENOMEM; goto done;}
|
||||
if(line[0] == LTAG) {
|
||||
char* url = ++line;
|
||||
char* rtag = strchr(line,RTAG);
|
||||
if(rtag == NULL) {
|
||||
nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line);
|
||||
free(triple);
|
||||
continue;
|
||||
}
|
||||
line = rtag + 1;
|
||||
@ -331,7 +362,6 @@ rccompile(const char* path)
|
||||
if(uri) ncurifree(uri);
|
||||
if(ncuriparse(url,&uri)) {
|
||||
nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line);
|
||||
free(triple);
|
||||
continue;
|
||||
}
|
||||
ncbytesclear(tmp);
|
||||
@ -341,9 +371,9 @@ rccompile(const char* path)
|
||||
ncbytescat(tmp,uri->port);
|
||||
}
|
||||
ncbytesnull(tmp);
|
||||
triple->host = ncbytesextract(tmp);
|
||||
if(strlen(triple->host)==0)
|
||||
{free(triple->host); triple->host = NULL;}
|
||||
host = ncbytesextract(tmp);
|
||||
if(strlen(host)==0)
|
||||
{free(host); host = NULL;}
|
||||
}
|
||||
/* split off key and value */
|
||||
key=line;
|
||||
@ -354,16 +384,30 @@ rccompile(const char* path)
|
||||
*value = '\0';
|
||||
value++;
|
||||
}
|
||||
triple->key = strdup(key);
|
||||
triple->value = strdup(value);
|
||||
/* See if key already exists */
|
||||
triple = rclocate(key,host);
|
||||
if(triple != NULL) {
|
||||
nullfree(triple->host);
|
||||
nullfree(triple->key);
|
||||
nullfree(triple->value);
|
||||
} else {
|
||||
triple = (NCTriple*)calloc(1,sizeof(NCTriple));
|
||||
if(triple == NULL) {ret = NC_ENOMEM; goto done;}
|
||||
nclistpush(rc,triple);
|
||||
}
|
||||
triple->host = host; host = NULL;
|
||||
triple->key = nulldup(key);
|
||||
triple->value = nulldup(value);
|
||||
rctrim(triple->host);
|
||||
rctrim(triple->key);
|
||||
rctrim(triple->value);
|
||||
|
||||
#ifdef D4DEBUG
|
||||
fprintf(stderr,"rc: host=%s key=%s value=%s\n",
|
||||
(triple->host != NULL ? triple->host : "<null>"),
|
||||
triple->key,triple->valu);
|
||||
#endif
|
||||
nclistpush(rc,triple);
|
||||
|
||||
triple = NULL;
|
||||
}
|
||||
rcorder(rc);
|
||||
@ -458,9 +502,13 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport)
|
||||
int ret = NC_NOERR;
|
||||
/* See if this key already defined */
|
||||
struct NCTriple* triple = NULL;
|
||||
NCRCglobalstate* globalstate = ncrc_getglobalstate();
|
||||
NClist* rc = globalstate->rcinfo.triples;
|
||||
NCRCglobalstate* globalstate = NULL;
|
||||
NClist* rc = NULL;
|
||||
|
||||
if(!NCRCinitialized) ncrc_initialize();
|
||||
globalstate = ncrc_getglobalstate();
|
||||
rc = globalstate->rcinfo.triples;
|
||||
|
||||
if(rc == NULL) {
|
||||
rc = nclistnew();
|
||||
if(rc == NULL) {ret = NC_ENOMEM; goto done;}
|
||||
|
||||
@ -388,7 +388,7 @@ From Zarr V2 Specification:
|
||||
a key formed from the index of the chunk within the grid of
|
||||
chunks representing the array. To form a string key for a
|
||||
chunk, the indices are converted to strings and concatenated
|
||||
with the period character (".") separating each index. For
|
||||
with the dimension_separator character ('/' or '.') separating each index. For
|
||||
example, given an array with shape (10000, 10000) and chunk
|
||||
shape (1000, 1000) there will be 100 chunks laid out in a 10 by
|
||||
10 grid. The chunk with indices (0, 0) provides data for rows
|
||||
@ -374,6 +374,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
|
||||
int i,stat = NC_NOERR;
|
||||
const char* value = NULL;
|
||||
NClist* modelist = nclistnew();
|
||||
int noflags = 0; /* track non-default negative flags */
|
||||
|
||||
if((value = controllookup((const char**)zinfo->envv_controls,"mode")) != NULL) {
|
||||
if((stat = NCZ_comma_parse(value,modelist))) goto done;
|
||||
@ -382,12 +383,20 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
|
||||
zinfo->controls.mapimpl = NCZM_DEFAULT;
|
||||
for(i=0;i<nclistlength(modelist);i++) {
|
||||
const char* p = nclistget(modelist,i);
|
||||
if(strcasecmp(p,PUREZARRCONTROL)==0) zinfo->controls.flags |= FLAG_PUREZARR;
|
||||
if(strcasecmp(p,PUREZARRCONTROL)==0) zinfo->controls.flags |= (FLAG_PUREZARR|FLAG_XARRAYDIMS);
|
||||
else if(strcasecmp(p,XARRAYCONTROL)==0) zinfo->controls.flags |= (FLAG_XARRAYDIMS|FLAG_PUREZARR); /*xarray=>zarr*/
|
||||
else if(strcasecmp(p,NOXARRAYCONTROL)==0) {
|
||||
noflags |= FLAG_XARRAYDIMS;
|
||||
zinfo->controls.flags |= FLAG_PUREZARR; /*noxarray=>zarr*/
|
||||
}
|
||||
else if(strcasecmp(p,"zip")==0) zinfo->controls.mapimpl = NCZM_ZIP;
|
||||
else if(strcasecmp(p,"file")==0) zinfo->controls.mapimpl = NCZM_FILE;
|
||||
else if(strcasecmp(p,"s3")==0) zinfo->controls.mapimpl = NCZM_S3;
|
||||
}
|
||||
/* Apply negative controls by turning off negative flags */
|
||||
/* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */
|
||||
zinfo->controls.flags &= (~noflags);
|
||||
|
||||
/* Process other controls */
|
||||
if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) {
|
||||
zinfo->controls.flags |= FLAG_LOGGING;
|
||||
|
||||
@ -12,6 +12,8 @@
|
||||
#ifndef ZARR_H
|
||||
#define ZARR_H
|
||||
|
||||
struct ChunkKey;
|
||||
|
||||
/* zarr.c */
|
||||
extern int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls);
|
||||
extern int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls);
|
||||
@ -56,7 +58,7 @@ extern int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
|
||||
extern int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
|
||||
extern int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
|
||||
extern int NCZ_isLittleEndian(void);
|
||||
extern int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist);
|
||||
extern int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);
|
||||
extern int NCZ_grpname_full(int gid, char** pathp);
|
||||
extern int ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var);
|
||||
extern int NCZ_comma_parse(const char* s, NClist* list);
|
||||
@ -65,6 +67,9 @@ extern char** NCZ_clonestringvec(size_t len, const char** vec);
|
||||
extern void NCZ_freestringvec(size_t len, char** vec);
|
||||
extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp);
|
||||
extern int NCZ_s3clear(ZS3INFO* s3);
|
||||
extern int NCZ_ischunkname(const char* name,char dimsep);
|
||||
extern char* NCZ_chunkpath(struct ChunkKey key,char dimsep);
|
||||
|
||||
/* Export */
|
||||
EXTERNL int NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3);
|
||||
|
||||
|
||||
@ -16,7 +16,10 @@ typedef struct NCZCacheEntry {
|
||||
struct List {void* next; void* prev; void* unused;} list;
|
||||
int modified;
|
||||
size64_t indices[NC_MAX_VAR_DIMS];
|
||||
char* key;
|
||||
struct ChunkKey {
|
||||
char* varkey; /* key to the containing variable */
|
||||
char* chunkkey; /* name of the chunk */
|
||||
} key;
|
||||
size64_t hashkey;
|
||||
void* data;
|
||||
} NCZCacheEntry;
|
||||
@ -29,20 +32,20 @@ typedef struct NCZChunkCache {
|
||||
size_t maxentries; /* Max number of entries allowed */
|
||||
NClist* mru; /* all cache entries in mru order */
|
||||
struct NCxcache* xcache;
|
||||
char dimension_separator;
|
||||
} NCZChunkCache;
|
||||
|
||||
/**************************************************/
|
||||
|
||||
extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption);
|
||||
extern int NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
|
||||
|
||||
extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, NCZChunkCache** cachep);
|
||||
extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, char dimsep, NCZChunkCache** cachep);
|
||||
extern void NCZ_free_chunk_cache(NCZChunkCache* cache);
|
||||
extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap);
|
||||
extern int NCZ_flush_chunk_cache(NCZChunkCache* cache);
|
||||
extern size64_t NCZ_cache_entrysize(NCZChunkCache* cache);
|
||||
extern NCZCacheEntry* NCZ_cache_entry(NCZChunkCache* cache, const size64_t* indices);
|
||||
extern size64_t NCZ_cache_size(NCZChunkCache* cache);
|
||||
extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp);
|
||||
extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key);
|
||||
|
||||
#endif /*ZCACHE_H*/
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#undef ZDEBUG1 /* detailed debug */
|
||||
|
||||
#undef ZCATCH /* Warning: significant performance impact */
|
||||
#undef ZTRACING /* Warning: significant performance impact */
|
||||
#define ZTRACING /* Warning: significant performance impact */
|
||||
|
||||
#include "ncexternl.h"
|
||||
#include "nclog.h"
|
||||
|
||||
@ -58,8 +58,26 @@ set_auto(void* func, void *client_data)
|
||||
int
|
||||
NCZ_initialize_internal(void)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char* dimsep = NULL;
|
||||
NCRCglobalstate* ngs = NULL;
|
||||
|
||||
ncz_initialized = 1;
|
||||
return NC_NOERR;
|
||||
/* Load the .rc file */
|
||||
if((stat=NC_rcload())) goto done;
|
||||
ngs = ncrc_getglobalstate();
|
||||
if(ngs != NULL) {
|
||||
/* Defaults */
|
||||
ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR;
|
||||
dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL);
|
||||
if(dimsep != NULL) {
|
||||
/* Verify its value */
|
||||
if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0]))
|
||||
ngs->zarr.dimension_separator = dimsep[0];
|
||||
}
|
||||
}
|
||||
done:
|
||||
return stat;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -57,6 +57,13 @@
|
||||
|
||||
#define PUREZARRCONTROL "zarr"
|
||||
#define XARRAYCONTROL "xarray"
|
||||
#define NOXARRAYCONTROL "noxarray"
|
||||
|
||||
#define LEGAL_DIM_SEPARATORS "./"
|
||||
#define DFALT_DIM_SEPARATOR '.'
|
||||
|
||||
#define islegaldimsep(c) ((c) != '\0' && strchr(LEGAL_DIM_SEPARATORS,(c)) != NULL)
|
||||
|
||||
|
||||
/* Mnemonics */
|
||||
#define ZCLOSE 1 /* this is closeorabort as opposed to enddef */
|
||||
@ -112,7 +119,7 @@ typedef struct NCZ_DIM_INFO {
|
||||
NCZcommon common;
|
||||
} NCZ_DIM_INFO_T;
|
||||
|
||||
/** Strut to hold ZARR-specific info for attributes. */
|
||||
/** Struct to hold ZARR-specific info for attributes. */
|
||||
typedef struct NCZ_ATT_INFO {
|
||||
NCZcommon common;
|
||||
} NCZ_ATT_INFO_T;
|
||||
@ -143,6 +150,7 @@ typedef struct NCZ_VAR_INFO {
|
||||
size_t scalar;
|
||||
struct NCZChunkCache* cache;
|
||||
struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */
|
||||
char dimension_separator; /* '.' | '/' */
|
||||
} NCZ_VAR_INFO_T;
|
||||
|
||||
/* Struct to hold ZARR-specific info for a field. */
|
||||
|
||||
@ -154,10 +154,27 @@ nczmap_write(NCZMAP* map, const char* key, size64_t start, size64_t count, const
|
||||
return map->api->write(map, key, start, count, content);
|
||||
}
|
||||
|
||||
/* Define a static qsort comparator for strings for use with qsort */
|
||||
static int
|
||||
cmp_strings(const void* a1, const void* a2)
|
||||
{
|
||||
const char** s1 = (const char**)a1;
|
||||
const char** s2 = (const char**)a2;
|
||||
return strcmp(*s1,*s2);
|
||||
}
|
||||
|
||||
int
|
||||
nczmap_search(NCZMAP* map, const char* prefix, NClist* matches)
|
||||
{
|
||||
return map->api->search(map, prefix, matches);
|
||||
int stat = NC_NOERR;
|
||||
if((stat = map->api->search(map, prefix, matches)) == NC_NOERR) {
|
||||
/* sort the list */
|
||||
if(nclistlength(matches) > 1) {
|
||||
void* base = nclistcontents(matches);
|
||||
qsort(base, nclistlength(matches), sizeof(char*), cmp_strings);
|
||||
}
|
||||
}
|
||||
return stat;
|
||||
}
|
||||
|
||||
/**************************************************/
|
||||
|
||||
@ -155,7 +155,7 @@ NCZM_S3=3, /* Amazon S3 implementation */
|
||||
} NCZM_IMPL;
|
||||
|
||||
/* Define the default map implementation */
|
||||
#define NCZM_DEFAULT NCZM_ZIP
|
||||
#define NCZM_DEFAULT NCZM_FILE
|
||||
|
||||
/* Define the per-implementation limitations flags */
|
||||
typedef size64_t NCZM_FEATURES;
|
||||
|
||||
@ -647,9 +647,11 @@ platformtestcontentbearing(ZFMAP* zfmap, const char* truepath)
|
||||
|
||||
/* Localize */
|
||||
if((ret = nczm_localize(truepath,&local,LOCALIZE))) goto done;
|
||||
|
||||
|
||||
errno = 0;
|
||||
if((ret = stat(local, &buf)) < 0) {
|
||||
ret = stat(local, &buf);
|
||||
ZTRACEMORE(6,"stat: local=%s ret=%d, errno=%d st_mode=%d",local,ret,errno,buf.st_mode);
|
||||
if(ret < 0) {
|
||||
ret = platformerr(errno);
|
||||
} else if(S_ISDIR(buf.st_mode)) {
|
||||
ret = NC_EEMPTY;
|
||||
|
||||
@ -323,7 +323,7 @@ zipread(NCZMAP* map, const char* key, size64_t start, size64_t count, void* cont
|
||||
char* buffer = NULL;
|
||||
char* truekey = NULL;
|
||||
zip_int64_t red = 0;
|
||||
|
||||
|
||||
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
|
||||
|
||||
switch(stat = zzlookupobj(zzmap,key,&zindex)) {
|
||||
@ -382,7 +382,7 @@ zipwrite(NCZMAP* map, const char* key, size64_t start, size64_t count, const voi
|
||||
zip_int32_t compression = 0;
|
||||
zip_error_t zerror;
|
||||
void* localbuffer = NULL;
|
||||
|
||||
|
||||
ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count);
|
||||
|
||||
zip_error_init(&zerror);
|
||||
|
||||
@ -368,6 +368,17 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
|
||||
if((stat = NCJappend(jvar,jtmp))) goto done;
|
||||
jtmp = NULL;
|
||||
|
||||
/* dimension_separator key */
|
||||
/* Single char defining the separator in chunk keys */
|
||||
if(zvar->dimension_separator != DFALT_DIM_SEPARATOR) {
|
||||
char sep[2];
|
||||
sep[0] = zvar->dimension_separator;/* make separator a string*/
|
||||
sep[1] = '\0';
|
||||
if((stat = NCJnewstring(NCJ_STRING,sep,&jtmp))) goto done;
|
||||
if((stat = NCJinsert(jvar,"dimension_separator",jtmp))) goto done;
|
||||
jtmp = NULL;
|
||||
}
|
||||
|
||||
/* build .zarray path */
|
||||
if((stat = nczm_concat(fullpath,ZARRAY,&key)))
|
||||
goto done;
|
||||
@ -1380,6 +1391,22 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
{stat = THROW(NC_ENOMEM); goto done;}
|
||||
if((stat = decodeints(jvalue, shapes))) goto done;
|
||||
}
|
||||
/* Capture dimension_separator (must precede chunk cache creation) */
|
||||
{
|
||||
NCRCglobalstate* ngs = ncrc_getglobalstate();
|
||||
assert(ngs != NULL);
|
||||
zvar->dimension_separator = 0;
|
||||
if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done;
|
||||
if(jvalue != NULL) {
|
||||
/* Verify its value */
|
||||
if(jvalue->sort == NCJ_STRING && jvalue->value != NULL && strlen(jvalue->value) == 1)
|
||||
zvar->dimension_separator = jvalue->value[0];
|
||||
}
|
||||
/* If value is invalid, then use global default */
|
||||
if(!islegaldimsep(zvar->dimension_separator))
|
||||
zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */
|
||||
assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */
|
||||
}
|
||||
/* chunks */
|
||||
{
|
||||
int rank;
|
||||
@ -1406,7 +1433,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
|
||||
}
|
||||
zvar->chunksize = zvar->chunkproduct * var->type_info->size;
|
||||
/* Create the cache */
|
||||
if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,&zvar->cache)))
|
||||
if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache)))
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
@ -1970,17 +1997,18 @@ computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int purezarr, int xarra
|
||||
/* xarray => purezarr */
|
||||
assert(!xarray || purezarr);
|
||||
|
||||
if(xarray) {/* Read in the attributes to get xarray dimdef attribute */
|
||||
if(xarray) {/* Read in the attributes to get xarray dimdef attribute; Note that it might not exist */
|
||||
char zdimname[4096];
|
||||
if(zvar->xarray == NULL) {
|
||||
assert(nclistlength(dimnames) == 0);
|
||||
if((stat = ncz_read_atts(file,(NC_OBJ*)var))) goto done;
|
||||
}
|
||||
assert(zvar->xarray != NULL);
|
||||
/* convert xarray to the dimnames */
|
||||
for(i=0;i<nclistlength(zvar->xarray);i++) {
|
||||
snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->xarray,i));
|
||||
nclistpush(dimnames,strdup(zdimname));
|
||||
if(zvar->xarray != NULL) {
|
||||
/* convert xarray to the dimnames */
|
||||
for(i=0;i<nclistlength(zvar->xarray);i++) {
|
||||
snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->xarray,i));
|
||||
nclistpush(dimnames,strdup(zdimname));
|
||||
}
|
||||
}
|
||||
createdims = 1; /* may need to create them */
|
||||
}
|
||||
|
||||
@ -451,7 +451,7 @@ is an object in the map.
|
||||
Note: need to test with "/", "", and with and without trailing "/".
|
||||
*/
|
||||
int
|
||||
NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist)
|
||||
NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist)
|
||||
{
|
||||
int i,stat=NC_NOERR;
|
||||
NClist* matches = nclistnew();
|
||||
@ -460,7 +460,6 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist
|
||||
/* Get the list of names just below prefix */
|
||||
if((stat = nczmap_search(map,prefix,matches))) goto done;
|
||||
for(i=0;i<nclistlength(matches);i++) {
|
||||
const char* p;
|
||||
const char* name = nclistget(matches,i);
|
||||
size_t namelen= strlen(name);
|
||||
/* Ignore keys that start with .z or .nc or a potential chunk name */
|
||||
@ -468,10 +467,8 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist
|
||||
continue;
|
||||
if(namelen >= 2 && name[0] == '.' && name[1] == 'z')
|
||||
continue;
|
||||
for(p=name;*p;p++) {
|
||||
if(*p != '.' && strchr("0123456789",*p) == NULL) break;
|
||||
}
|
||||
if(*p == '\0') continue; /* looks like a chunk name */
|
||||
if(NCZ_ischunkname(name,dimsep))
|
||||
continue;
|
||||
/* Create <prefix>/<name>/<tag> and see if it exists */
|
||||
ncbytesclear(path);
|
||||
ncbytescat(path,prefix);
|
||||
@ -878,3 +875,32 @@ endswith(const char* s, const char* suffix)
|
||||
return 1;
|
||||
}
|
||||
|
||||
int
|
||||
NCZ_ischunkname(const char* name,char dimsep)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
const char* p;
|
||||
if(strchr("0123456789",name[0])== NULL)
|
||||
stat = NC_ENCZARR;
|
||||
else for(p=name;*p;p++) {
|
||||
if(*p != dimsep && strchr("0123456789",*p) == NULL) /* approximate */
|
||||
{stat = NC_ENCZARR; break;}
|
||||
}
|
||||
return stat;
|
||||
}
|
||||
|
||||
char*
|
||||
NCZ_chunkpath(struct ChunkKey key,char dimsep)
|
||||
{
|
||||
size_t plen = nulllen(key.varkey)+1+nulllen(key.chunkkey);
|
||||
char* path = (char*)malloc(plen+1);
|
||||
char sdimsep[2];
|
||||
|
||||
if(path == NULL) return NULL;
|
||||
path[0] = '\0';
|
||||
strlcat(path,key.varkey,plen+1);
|
||||
sdimsep[0] = dimsep; sdimsep[1] = '\0';
|
||||
strlcat(path,sdimsep,plen+1);
|
||||
strlcat(path,key.chunkkey,plen+1);
|
||||
return path;
|
||||
}
|
||||
|
||||
@ -381,6 +381,9 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
|
||||
zvar->common.file = h5;
|
||||
zvar->scalar = (ndims == 0 ? 1 : 0);
|
||||
|
||||
zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator;
|
||||
assert(zvar->dimension_separator != 0);
|
||||
|
||||
/* Set these state flags for the var. */
|
||||
var->is_new_var = NC_TRUE;
|
||||
var->meta_read = NC_TRUE;
|
||||
@ -455,7 +458,7 @@ var->type_info->rc++;
|
||||
var->chunk_cache_preemption = 1; /* not used */
|
||||
|
||||
/* Create the cache */
|
||||
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,&zvar->cache)))
|
||||
if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache)))
|
||||
BAIL(retval);
|
||||
|
||||
/* Is this a variable with a chunksize greater than the current cache size? */
|
||||
|
||||
@ -144,7 +144,7 @@ NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
|
||||
* @author Dennis Heimbigner, Ed Hartnett
|
||||
*/
|
||||
int
|
||||
NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** cachep)
|
||||
NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZChunkCache** cachep)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
NCZChunkCache* cache = NULL;
|
||||
@ -163,6 +163,7 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** c
|
||||
assert(cache->fillchunk == NULL);
|
||||
cache->fillchunk = NULL;
|
||||
cache->chunksize = chunksize;
|
||||
cache->dimension_separator = dimsep;
|
||||
|
||||
/* Figure out the actual cache size */
|
||||
cachesize = var->chunk_cache_size;
|
||||
@ -203,7 +204,7 @@ NCZ_free_chunk_cache(NCZChunkCache* cache)
|
||||
NCZCacheEntry* entry = nclistremove(cache->mru,0);
|
||||
(void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr);
|
||||
assert(ptr == entry);
|
||||
nullfree(entry->data); nullfree(entry->key); nullfree(entry);
|
||||
nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey); nullfree(entry);
|
||||
}
|
||||
#ifdef DEBUG
|
||||
fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru));
|
||||
@ -297,7 +298,7 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru));
|
||||
|
||||
done:
|
||||
if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */
|
||||
if(entry) {nullfree(entry->data); nullfree(entry->key);}
|
||||
if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);}
|
||||
nullfree(entry);
|
||||
return THROW(stat);
|
||||
}
|
||||
@ -333,7 +334,7 @@ fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru));
|
||||
entry = NULL;
|
||||
|
||||
done:
|
||||
if(entry) {nullfree(entry->data); nullfree(entry->key);}
|
||||
if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);}
|
||||
nullfree(entry);
|
||||
return THROW(stat);
|
||||
}
|
||||
@ -359,7 +360,7 @@ makeroom(NCZChunkCache* cache)
|
||||
if(e->modified) /* flush to file */
|
||||
stat=put_chunk(cache,e);
|
||||
/* reclaim */
|
||||
nullfree(e->data); nullfree(e->key); nullfree(e);
|
||||
nullfree(e->data); nullfree(e->key.varkey); nullfree(e->key.chunkkey); nullfree(e);
|
||||
}
|
||||
#ifdef DEBUG
|
||||
fprintf(stderr,"|cache.makeroom|=%ld\n",nclistlength(cache->mru));
|
||||
@ -423,32 +424,36 @@ From Zarr V2 Specification:
|
||||
a key formed from the index of the chunk within the grid of
|
||||
chunks representing the array. To form a string key for a
|
||||
chunk, the indices are converted to strings and concatenated
|
||||
with the period character (".") separating each index. For
|
||||
example, given an array with shape (10000, 10000) and chunk
|
||||
shape (1000, 1000) there will be 100 chunks laid out in a 10 by
|
||||
10 grid. The chunk with indices (0, 0) provides data for rows
|
||||
0-1000 and columns 0-1000 and is stored under the key "0.0"; the
|
||||
chunk with indices (2, 4) provides data for rows 2000-3000 and
|
||||
columns 4000-5000 and is stored under the key "2.4"; etc."
|
||||
with the dimension_separator character ('.' or '/') separating
|
||||
each index. For example, given an array with shape (10000,
|
||||
10000) and chunk shape (1000, 1000) there will be 100 chunks
|
||||
laid out in a 10 by 10 grid. The chunk with indices (0, 0)
|
||||
provides data for rows 0-1000 and columns 0-1000 and is stored
|
||||
under the key "0.0"; the chunk with indices (2, 4) provides data
|
||||
for rows 2000-3000 and columns 4000-5000 and is stored under the
|
||||
key "2.4"; etc."
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param R Rank
|
||||
* @param chunkindices The chunk indices
|
||||
* @param dimsep the dimension separator
|
||||
* @param keyp Return the chunk key string
|
||||
*/
|
||||
int
|
||||
NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp)
|
||||
NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char dimsep, char** keyp)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
int r;
|
||||
NCbytes* key = ncbytesnew();
|
||||
|
||||
if(keyp) *keyp = NULL;
|
||||
|
||||
assert(islegaldimsep(dimsep));
|
||||
|
||||
for(r=0;r<R;r++) {
|
||||
char sindex[64];
|
||||
if(r > 0) ncbytescat(key,".");
|
||||
if(r > 0) ncbytesappend(key,dimsep);
|
||||
/* Print as decimal with no leading zeros */
|
||||
snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]);
|
||||
ncbytescat(key,sindex);
|
||||
@ -485,7 +490,11 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry)
|
||||
zfile = ((cache->var->container)->nc4_info)->format_file_info;
|
||||
map = zfile->map;
|
||||
|
||||
stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data);
|
||||
{
|
||||
char* path = NCZ_chunkpath(entry->key,cache->dimension_separator);
|
||||
stat = nczmap_write(map,path,0,cache->chunksize,entry->data);
|
||||
nullfree(path);
|
||||
}
|
||||
switch(stat) {
|
||||
case NC_NOERR:
|
||||
break;
|
||||
@ -514,7 +523,7 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
|
||||
NC_FILE_INFO_T* file = NULL;
|
||||
NCZ_FILE_INFO_T* zfile = NULL;
|
||||
|
||||
ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key);
|
||||
ZTRACE(5,"cache.var=%s entry.key=%s sep=%d",cache->var->hdr.name,entry->key,cache->dimension_separator);
|
||||
|
||||
LOG((3, "%s: file: %p", __func__, file));
|
||||
|
||||
@ -523,32 +532,33 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry)
|
||||
map = zfile->map;
|
||||
assert(map && entry->data);
|
||||
|
||||
stat = nczmap_read(map,entry->key,0,cache->chunksize,(char*)entry->data);
|
||||
{
|
||||
char* path = NCZ_chunkpath(entry->key,cache->dimension_separator);
|
||||
stat = nczmap_read(map,path,0,cache->chunksize,(char*)entry->data);
|
||||
nullfree(path);
|
||||
}
|
||||
|
||||
return ZUNTRACE(stat);
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp)
|
||||
NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
char* chunkname = NULL;
|
||||
char* varkey = NULL;
|
||||
char* key = NULL;
|
||||
|
||||
assert(key != NULL);
|
||||
/* Get the chunk object name */
|
||||
if((stat = NCZ_buildchunkkey(cache->ndims, chunkindices, &chunkname))) goto done;
|
||||
if((stat = NCZ_buildchunkkey(cache->ndims, chunkindices, cache->dimension_separator, &chunkname))) goto done;
|
||||
/* Get the var object key */
|
||||
if((stat = NCZ_varkey(cache->var,&varkey))) goto done;
|
||||
/* Prefix the path to the containing variable object */
|
||||
if((stat=nczm_concat(varkey,chunkname,&key))) goto done;
|
||||
if(keyp) {*keyp = key; key = NULL;}
|
||||
key->varkey = varkey; varkey = NULL;
|
||||
key->chunkkey = chunkname; chunkname = NULL;
|
||||
|
||||
done:
|
||||
nullfree(chunkname);
|
||||
nullfree(varkey);
|
||||
nullfree(key);
|
||||
return THROW(stat);
|
||||
}
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ HDF5 Support: @HAS_HDF5@
|
||||
NetCDF-4 API: @HAS_NC4@
|
||||
NC-4 Parallel Support: @HAS_PARALLEL4@
|
||||
PnetCDF Support: @HAS_PNETCDF@
|
||||
DAP2 Support: @HAS_DAP@
|
||||
DAP2 Support: @HAS_DAP2@
|
||||
DAP4 Support: @HAS_DAP4@
|
||||
Byte-Range Support: @HAS_BYTERANGE@
|
||||
Diskless Support: @HAS_DISKLESS@
|
||||
|
||||
@ -3,64 +3,48 @@
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
# Enable if using localhost
|
||||
LOCAL=1
|
||||
|
||||
RCEMBED=1
|
||||
RCLOCAL=1
|
||||
RCHOME=1
|
||||
RCENV=1
|
||||
RCPREC=1
|
||||
|
||||
# Not currently testable in netcdf
|
||||
#RCSPEC=1
|
||||
|
||||
#SHOW=1
|
||||
#DBG=1
|
||||
|
||||
# Choose at most 1
|
||||
#GDB=1
|
||||
#VG=1
|
||||
|
||||
NFL=1
|
||||
if test "x$NCAUTH_HOMETEST" != x ; then
|
||||
RCHOME=1
|
||||
fi
|
||||
RCHOME=1
|
||||
|
||||
WD=`pwd`
|
||||
|
||||
NETRCFILE=$WD/test_auth_netrc
|
||||
# This is the control variable; set when needed
|
||||
unset NETRC
|
||||
|
||||
COOKIES="${WD}/test_auth_cookies"
|
||||
COOKIES="${WD}/.cookies_test"
|
||||
|
||||
RC=.daprc
|
||||
|
||||
OCLOGFILE=stderr
|
||||
if test "x$FPISMSVC" = x ; then
|
||||
NETRC=.netrc_test
|
||||
NETRCIMP=.netrc
|
||||
else
|
||||
NETRC=_netrc_test
|
||||
NETRCIMP=_netrc
|
||||
fi
|
||||
|
||||
LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP"
|
||||
HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP"
|
||||
|
||||
NETRCFILE=$WD/$NETRC
|
||||
DAPRCFILE=$WD/$RC
|
||||
|
||||
HOMENETRCFILE=$HOME/$NETRC
|
||||
HOMEDAPRCFILE=$HOME/$RC
|
||||
|
||||
if test "x$DBG" = x1 ; then
|
||||
SHOW=1
|
||||
fi
|
||||
|
||||
# Major parameters
|
||||
|
||||
BASICCOMBO="tiggeUser:tigge"
|
||||
BADCOMBO="tiggeUser:xxxxx"
|
||||
URLPATH="thredds/dodsC/testRestrictedDataset/testData2.nc"
|
||||
PROTO=http
|
||||
if test "x$LOCAL" = x ; then
|
||||
URLSERVER="remotetest.unidata.ucar.edu"
|
||||
else
|
||||
URLSERVER="localhost:8081"
|
||||
fi
|
||||
|
||||
# See if we need to override
|
||||
if test "x$URS" != "x" ; then
|
||||
#https://54.86.135.31/opendap/data/nc/fnoc1.nc.dds
|
||||
URLSERVER="54.86.135.31"
|
||||
URLPATH="opendap/data/nc/fnoc1.nc"
|
||||
BASICCOMBO="$URS"
|
||||
RCEMBED=0
|
||||
NETRC=$NETRCFILE
|
||||
AUTHSERVER="thredds.ucar.edu"
|
||||
BASICCOMBO="authtester:auth"
|
||||
URLPATH="thredds/dodsC/test3/testData.nc"
|
||||
PROTO=https
|
||||
fi
|
||||
URLSERVER=${AUTHSERVER}
|
||||
|
||||
if test "x$DBG" = x1 ; then
|
||||
URLPATH="${URLPATH}#log&show=fetch"
|
||||
@ -70,230 +54,154 @@ fi
|
||||
BASICUSER=`echo $BASICCOMBO | cut -d: -f1`
|
||||
BASICPWD=`echo $BASICCOMBO | cut -d: -f2`
|
||||
|
||||
OUTPUT="./.output"
|
||||
|
||||
if test "x$TEMP" = x ; then
|
||||
TEMP="/tmp"
|
||||
fi
|
||||
TEMP=`echo "$TEMP" | sed -e "s|/$||"`
|
||||
|
||||
LOCALRC=./$RC
|
||||
LOCALRC=${WD}/$RC
|
||||
LOCALNETRC=${WD}/$NETRC
|
||||
HOMERC=${HOME}/$RC
|
||||
HOMERC=`echo "$HOMERC" | sed -e "s|//|/|g"`
|
||||
SPECRC="$TEMP/temprc"
|
||||
ENVRC="$WD/envrc"
|
||||
HOMENETRC=${HOME}/$NETRC
|
||||
HOMENETRC=`echo "$HOMENETRC" | sed -e "s|//|/|g"`
|
||||
|
||||
createrc() {
|
||||
RCP="$1" ; shift
|
||||
unset NOPWD
|
||||
unset BADPWD
|
||||
while [[ $# > 0 ]] ; do
|
||||
case "$1" in
|
||||
nopwd) NOPWD=1 ;;
|
||||
badpwd) BADPWD=1 ;;
|
||||
*) ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
if test "x$RCP" != x ; then
|
||||
rm -f $RCP
|
||||
echo "Creating rc file $RCP"
|
||||
else
|
||||
local RCP
|
||||
local NETRCFILE
|
||||
RCP="$1"
|
||||
if test "x$RCP" = x ; then
|
||||
echo "createrc: no rc specified"
|
||||
exit 1
|
||||
fi
|
||||
if test "x${DBG}" != x ; then
|
||||
shift
|
||||
NETRCPATH="$1"
|
||||
echo "Creating rc file $RCP"
|
||||
if test "x${DBG}" = x1 ; then
|
||||
echo "HTTP.VERBOSE=1" >>$RCP
|
||||
fi
|
||||
if test "x$NETRCPATH" = x ; then
|
||||
echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP
|
||||
elif test "x$NETRCPATH" != ximplicit ; then
|
||||
echo "HTTP.NETRC=${NETRCPATH}" >>$RCP
|
||||
elif test "x$NETRCPATH" = ximplicit ; then
|
||||
echo "HTTP.NETRC=" >>$RCP
|
||||
fi
|
||||
echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP
|
||||
if test "x${URS}" = x ; then
|
||||
if test "x${NOPWD}" = x ; then
|
||||
if test "x${BADPWD}" = x ; then
|
||||
echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP
|
||||
else
|
||||
echo "HTTP.CREDENTIALS.USERPASSWORD=${BADCOMBO}" >>$RCP
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if test "x${NETRC}" != x && test "x$NFL" = x ; then
|
||||
echo "HTTP.NETRC=${NETRC}" >>$RCP
|
||||
fi
|
||||
}
|
||||
|
||||
createnetrc() {
|
||||
NCP="$1" ; shift
|
||||
unset NOPWD
|
||||
unset BADPWD
|
||||
while [[ $# > 0 ]] ; do
|
||||
case "$1" in
|
||||
nopwd) NOPWD=1 ;;
|
||||
badpwd) BADPWD=1 ;;
|
||||
*) ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
if test "x$NCP" != x ; then
|
||||
rm -f $NCP
|
||||
echo "Creating netrc file $NCP"
|
||||
else
|
||||
echo "createnetrc: no rc specified"
|
||||
exit 1
|
||||
fi
|
||||
if test "x$URS" != x ; then
|
||||
echo "machine uat.urs.earthdata.nasa.gov login $BASICUSER password $BASICPWD" >>$NCP
|
||||
#echo "machine 54.86.135.31 login $BASICUSER password $BASICPWD" >>$1
|
||||
else
|
||||
echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NCP
|
||||
if test "x$NOPWD" = x ; then
|
||||
if test "x$BADPWD" = x ; then
|
||||
echo -n " login $BASICUSER password $BASICPWD" >>$NCP
|
||||
else
|
||||
echo -n " login $BASICUSER password xxxxxx" >>$NCP
|
||||
fi
|
||||
fi
|
||||
echo "" >>$NCP
|
||||
fi
|
||||
local NETRCPATH
|
||||
NETRCPATH="$1"; # netrc file path
|
||||
if test "x$NETRCPATH" = x ; then return; fi
|
||||
echo "Creating netrc file $NETRCPATH"
|
||||
# echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NETRCPATH
|
||||
echo -n "machine $URLSERVER" >>$NETRCPATH
|
||||
echo -n " login $BASICUSER password $BASICPWD" >>$NETRCPATH
|
||||
echo "" >>$NETRCPATH
|
||||
chmod go-rwx $NETRCPATH
|
||||
}
|
||||
|
||||
# Test cases
|
||||
|
||||
# Case: !daprc !netrc embedded usr:pwd
|
||||
rcembed() {
|
||||
echo "***Testing with embedded user:pwd"
|
||||
reset
|
||||
URL="${PROTO}://${BASICCOMBO}@${URLSERVER}/$URLPATH"
|
||||
# Invoke ncdump to extract a file the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: local daprc no netrc no embed
|
||||
rclocal1() {
|
||||
echo "***Testing rc file in local directory"
|
||||
reset
|
||||
# Create the rc file in ./
|
||||
createrc $LOCALRC
|
||||
# Invoke ncdump to extract a file using the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: local daprc local netrc no embed
|
||||
rclocal2() {
|
||||
echo "***Testing rc file in local directory"
|
||||
reset
|
||||
# Create the rc file and (optional) netrc fil in ./
|
||||
createnetrc $LOCALNETRC
|
||||
createrc $LOCALRC $LOCALNETRC
|
||||
# Invoke ncdump to extract a file using the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: home rc no netrc no embed
|
||||
rchome1() {
|
||||
echo "***Testing home rc file no netrc in home directory"
|
||||
reset
|
||||
# Create the rc file file in ./
|
||||
createrc $HOMERC
|
||||
# Invoke ncdump to extract a file the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: home daprc implicit home netrc no embed
|
||||
rchome2() {
|
||||
echo "***Testing .netrc file in home directory"
|
||||
reset
|
||||
createnetrc $HOME/$NETRCIMP
|
||||
createrc $HOMERC implicit
|
||||
# Invoke ncdump to extract a file using the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: local rc explicit netrc no embed
|
||||
rchome3() {
|
||||
echo "***Testing local rc file and .netrc explicit in home directory"
|
||||
reset
|
||||
# Create the rc file and (optional) netrc file in ./
|
||||
createnetrc $HOME/$NETRC
|
||||
createrc $LOCALRC $HOME/$NETRC
|
||||
# Invoke ncdump to extract a file the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
# Case: local rc implicit netrc no embed
|
||||
rchome4() {
|
||||
echo "***Testing local rc file and .netrc implicit in home directory"
|
||||
reset
|
||||
# Create the rc file and (optional) netrc file in ./
|
||||
createnetrc $HOME/$NETRCIMP
|
||||
createrc $LOCALRC implicit
|
||||
# Invoke ncdump to extract a file the URL
|
||||
${NCDUMP} -h "$URL" > testauthoutput
|
||||
}
|
||||
|
||||
reset() {
|
||||
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC $OUTPUT ; do
|
||||
rm -f ${f}
|
||||
done
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
rm -f $HOMERCFILES
|
||||
fi
|
||||
rm -f $LOCALRCFILES
|
||||
unset DAPRCFILE
|
||||
rm -f ./testauthoutput
|
||||
}
|
||||
|
||||
restore() {
|
||||
reset
|
||||
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do
|
||||
if test -f ${f}.save ; then
|
||||
echo "restoring old ${f}"
|
||||
cp ${f}.save ${f}
|
||||
fi
|
||||
done
|
||||
}
|
||||
rcembed
|
||||
|
||||
save() {
|
||||
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do
|
||||
if test -f $f ; then
|
||||
if test -f ${f}.save ; then
|
||||
ignore=1
|
||||
else
|
||||
echo "saving $f"
|
||||
cp ${f} ${f}.save
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show() {
|
||||
if test "x$SHOW" = x1 ; then cat $OUTPUT; fi
|
||||
if test "x$OUTPUT" != "x"; then rm -f $OUTPUT; fi
|
||||
}
|
||||
|
||||
# Assemble the ncdump command
|
||||
if test "x$DBG" = x1; then
|
||||
NCDUMP="$NCDUMP -D1"
|
||||
fi
|
||||
|
||||
if test "x$GDB" = x1 ; then
|
||||
NCDUMP="gdb --args $NCDUMP"
|
||||
fi
|
||||
if test "x$VG" = x1 ; then
|
||||
NCDUMP="valgrind --leak-check=full $NCDUMP"
|
||||
fi
|
||||
|
||||
# Initialize
|
||||
save
|
||||
reset
|
||||
|
||||
if test "x$RCEMBED" = x1 ; then
|
||||
echo "***Testing rc file with embedded user:pwd"
|
||||
URL="${PROTO}://${BASICCOMBO}@${URLSERVER}/$URLPATH"
|
||||
unset NETRC
|
||||
# Invoke ncdump to extract a file the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
show
|
||||
fi
|
||||
|
||||
# Rest of tests assume these defaults
|
||||
# Next set tests assume these defaults
|
||||
URL="${PROTO}://${URLSERVER}/$URLPATH"
|
||||
NETRC=$NETRCFILE
|
||||
|
||||
if test "x$RCLOCAL" = x1 ; then
|
||||
echo "***Testing rc file in local directory"
|
||||
# Create the rc file and (optional) netrc fil in ./
|
||||
reset
|
||||
createnetrc $NETRC
|
||||
createrc $LOCALRC
|
||||
|
||||
# Invoke ncdump to extract a file using the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
show
|
||||
fi
|
||||
rclocal1
|
||||
rclocal2
|
||||
|
||||
# Do not do this unless you know what you are doing
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
echo "***Testing rc file in home directory"
|
||||
# Create the rc file and (optional) netrc file in ./
|
||||
reset
|
||||
createnetrc $NETRC
|
||||
createrc $HOMERC
|
||||
|
||||
# Invoke ncdump to extract a file the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
show
|
||||
fi
|
||||
|
||||
if test "x$RCSPEC" == x1 ; then
|
||||
echo "*** Testing rc file in specified directory"
|
||||
# Create the rc file and (optional) netrc file
|
||||
reset
|
||||
createnetrc $NETRC
|
||||
createrc $SPECRC
|
||||
|
||||
# Invoke ncdump to extract a file the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
show
|
||||
fi
|
||||
|
||||
if test "x$RCENV" = x1 ; then
|
||||
echo "*** Testing rc file using env variable"
|
||||
# Create the rc file and (optional) netrc file
|
||||
reset
|
||||
createnetrc $NETRC
|
||||
echo "ENV: export DAPRCFILE=$ENVRC"
|
||||
export DAPRCFILE=$ENVRC
|
||||
createrc $DAPRCFILE
|
||||
|
||||
# Invoke ncdump to extract a file the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
show
|
||||
export DAPRCFILE=
|
||||
fi
|
||||
|
||||
# Test that .daprc overrides netcrc for password
|
||||
URL="${PROTO}://${URLSERVER}/$URLPATH"
|
||||
NETRC=$NETRCFILE
|
||||
if test "x$RCPREC" = x1 ; then
|
||||
echo "***Testing rc vs netrc file precedence"
|
||||
# Create the rc file and (optional) netrc file in ./
|
||||
reset
|
||||
createnetrc $NETRC badpwd
|
||||
createrc $LOCALRC
|
||||
|
||||
# Invoke ncdump to extract a file using the URL
|
||||
echo "command: ${NCDUMP} -h ${URL} > $OUTPUT"
|
||||
${NCDUMP} -h "$URL" > $OUTPUT
|
||||
${NCDUMP} -h "$URL"
|
||||
show
|
||||
rchome1
|
||||
rchome2
|
||||
rchome3
|
||||
rchome4
|
||||
fi
|
||||
|
||||
reset
|
||||
restore
|
||||
|
||||
exit
|
||||
|
||||
|
||||
@ -99,9 +99,12 @@ IF(ENABLE_TESTS)
|
||||
ADD_EXECUTABLE(bom bom.c)
|
||||
ADD_EXECUTABLE(tst_dimsizes tst_dimsizes.c)
|
||||
ADD_EXECUTABLE(nctrunc nctrunc.c)
|
||||
ADD_EXECUTABLE(tst_rcmerge tst_rcmerge.c)
|
||||
TARGET_LINK_LIBRARIES(rewrite-scalar netcdf)
|
||||
TARGET_LINK_LIBRARIES(bom netcdf)
|
||||
TARGET_LINK_LIBRARIES(tst_dimsizes netcdf)
|
||||
TARGET_LINK_LIBRARIES(nctrunc netcdf)
|
||||
TARGET_LINK_LIBRARIES(tst_rcmerge netcdf)
|
||||
|
||||
IF(USE_HDF5)
|
||||
ADD_EXECUTABLE(tst_fileinfo tst_fileinfo.c)
|
||||
@ -137,6 +140,13 @@ IF(ENABLE_TESTS)
|
||||
SET_TARGET_PROPERTIES(nctrunc PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
IF(USE_HDF5)
|
||||
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
@ -305,6 +315,8 @@ IF(ENABLE_TESTS)
|
||||
add_sh_test(ncdump test_keywords)
|
||||
ENDIF()
|
||||
|
||||
add_sh_test(ncdump test_rcmerge)
|
||||
|
||||
ENDIF(ENABLE_TESTS)
|
||||
|
||||
#IF(MSVC)
|
||||
|
||||
@ -61,14 +61,14 @@ man_MANS = ncdump.1 nccopy.1
|
||||
if BUILD_TESTSETS
|
||||
# C programs needed by shell scripts for classic tests.
|
||||
check_PROGRAMS = rewrite-scalar ref_ctest ref_ctest64 ncdump tst_utf8 \
|
||||
bom tst_dimsizes nctrunc
|
||||
bom tst_dimsizes nctrunc tst_rcmerge
|
||||
|
||||
# Tests for classic and 64-bit offset files.
|
||||
TESTS = tst_inttags.sh run_tests.sh tst_64bit.sh ref_ctest \
|
||||
ref_ctest64 tst_output.sh tst_lengths.sh tst_calendars.sh \
|
||||
run_utf8_tests.sh test_unicode_directory.sh tst_nccopy3.sh tst_nccopy3_subset.sh \
|
||||
tst_charfill.sh tst_iter.sh tst_formatx3.sh tst_bom.sh \
|
||||
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh
|
||||
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh test_rcmerge.sh
|
||||
|
||||
# The tst_nccopy3.sh test uses output from a bunch of other
|
||||
# tests. This records the dependency so parallel builds work.
|
||||
@ -164,7 +164,8 @@ ref_no_ncproperty.nc test_unicode_directory.sh \
|
||||
ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \
|
||||
test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl \
|
||||
ref_keyword3.cdl ref_keyword4.cdl \
|
||||
ref_tst_nofilters.cdl test_unicode_path.sh
|
||||
ref_tst_nofilters.cdl test_unicode_path.sh test_rcmerge.sh \
|
||||
ref_rcmerge1.txt ref_rcmerge2.txt ref_rcmerge3.txt
|
||||
|
||||
# The L512.bin file is file containing exactly 512 bytes each of value 0.
|
||||
# It is used for creating hdf5 files with varying offsets for testing.
|
||||
@ -176,7 +177,7 @@ EXTRA_DIST += tst_ctests.sh ref_ctest_small_3.c ref_ctest_small_4.c \
|
||||
# CDL files and Expected results
|
||||
SUBDIRS = cdl expected
|
||||
|
||||
CLEANFILES = tst_*.nc tmp*.nc test*.nc iter.* tmp*.cdl \
|
||||
CLEANFILES = tst_*.nc tmp*.nc test*.nc iter.* tmp*.cdl tmp*.txt \
|
||||
tst_output_*.cdl tst_output_*.c tst_utf8_*.cdl *.tmp tst_tst8.cdl \
|
||||
tst_netcdf4_*.cdl test1_ncdump.cdl test2_ncdump.cdl test1.cdl \
|
||||
ctest1.cdl test1_cdf5.cdl test2_cdf5.cdl test1_offset.cdl \
|
||||
|
||||
6
ncdump/ref_rcmerge1.txt
Normal file
6
ncdump/ref_rcmerge1.txt
Normal file
@ -0,0 +1,6 @@
|
||||
|ncrc_home|->|ncrc|
|
||||
|daprc_home|->|daprc|
|
||||
|dodsrc_home|->|dodsrc|
|
||||
|ncrc_local|->|ncrc|
|
||||
|daprc_local|->|daprc|
|
||||
|dodsrc_local|->|dodsrc|
|
||||
3
ncdump/ref_rcmerge2.txt
Normal file
3
ncdump/ref_rcmerge2.txt
Normal file
@ -0,0 +1,3 @@
|
||||
|ncrc|->|ncrc|
|
||||
|daprc|->|daprc|
|
||||
|dodsrc|->|dodsrc|
|
||||
3
ncdump/ref_rcmerge3.txt
Normal file
3
ncdump/ref_rcmerge3.txt
Normal file
@ -0,0 +1,3 @@
|
||||
|ncrc|->|ncrc2|
|
||||
|ncrcx|->|ncrcy|
|
||||
|daprc|->|daprc|
|
||||
93
ncdump/test_rcmerge.sh
Executable file
93
ncdump/test_rcmerge.sh
Executable file
@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
# Test that all available .rc files are merged and overrides are handled.
|
||||
# The internal .rc is constructed as follows:
|
||||
#
|
||||
# 1. Use NCRCENV_RC environment variable exclusively if defined
|
||||
# 2. If NCRCENV_RC is not defined then merge the set of rc files in this order:
|
||||
# 1. $HOME/.ncrc
|
||||
# 2. $HOME/.daprc
|
||||
# 3. $HOME/.docsrc
|
||||
# 4. $CWD/.ncrc
|
||||
# 5. $CWD/.daprc
|
||||
# 6. $CWD/.docsrc
|
||||
# Entries in later files override any of the >earlier >files
|
||||
|
||||
if test "x$NCAUTH_HOMETEST" != x ; then
|
||||
RCHOME=1
|
||||
fi
|
||||
|
||||
WD=`pwd`
|
||||
|
||||
HOMERCFILES="$HOME/.ncrc $HOME/.daprc $HOME/.dodsrc"
|
||||
LOCALRCFILES="$WD/.ncrc $WD/.daprc $WD/.dodsrc"
|
||||
|
||||
resetrc() {
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
rm -f $HOMERCFILES
|
||||
fi
|
||||
rm -f $LOCALRCFILES
|
||||
unset NCRCENV_RC
|
||||
}
|
||||
|
||||
mergecase1() {
|
||||
# create everything with different keys to test merge
|
||||
resetrc
|
||||
rm -f tmp_rcmerge.txt tmpoutput.txt
|
||||
for r in "ncrc" "daprc" "dodsrc" ; do
|
||||
if test "x$RCHOME" = x1 ; then echo "${r}_home=${r}" >> $HOME/".${r}"; fi
|
||||
echo "${r}_local=${r}" >> $WD/".${r}"
|
||||
done;
|
||||
${execdir}/tst_rcmerge > tmpoutput.txt
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
cp ${srcdir}/ref_rcmerge1.txt tmp_rcmerge1.txt
|
||||
else
|
||||
sed -e '/_local/p' -e d <${srcdir}/ref_rcmerge1.txt > tmp_rcmerge1.txt
|
||||
fi
|
||||
diff -b tmp_rcmerge1.txt tmpoutput.txt
|
||||
}
|
||||
|
||||
mergecase2() {
|
||||
# create with some same keys to test override
|
||||
resetrc
|
||||
rm -f tmp_rcmerge.txt tmpoutput.txt
|
||||
for r in "ncrc" "daprc" "dodsrc" ; do
|
||||
if test "x$RCHOME" = x1 ; then echo "${r}=${r}" >> $HOME/".${r}"; fi
|
||||
echo "${r}=${r}" >> $WD/".${r}"
|
||||
done;
|
||||
${execdir}/tst_rcmerge > tmpoutput.txt
|
||||
diff -b ${srcdir}/ref_rcmerge2.txt tmpoutput.txt
|
||||
}
|
||||
|
||||
mergecase3() {
|
||||
# Test cross file overrides
|
||||
resetrc
|
||||
rm -f tmp_rcmerge.txt tmpoutput.txt
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
echo "ncrc=ncrc1" >> $HOME/.ncrc
|
||||
echo "ncrcx=ncrcx" >> $HOME/.ncrc
|
||||
echo "ncrc=ncrc2" >> $HOME/.dodsrc
|
||||
echo "daprc=daprc" >> $HOME/.daprc
|
||||
else
|
||||
echo "ncrc=ncrc1" >> $WD/.ncrc
|
||||
echo "ncrcx=ncrcx" >> $WD/.ncrc
|
||||
echo "ncrc=ncrc2" >> $WD/.dodsrc
|
||||
echo "daprc=daprc" >> $WD/.daprc
|
||||
fi
|
||||
echo "daprc=daprc" >> $WD/.dodsrc
|
||||
echo "ncrcx=ncrcy" >> $WD/.dodsrc
|
||||
|
||||
${execdir}/tst_rcmerge > tmpoutput.txt
|
||||
diff -b ${srcdir}/ref_rcmerge3.txt tmpoutput.txt
|
||||
}
|
||||
|
||||
resetrc
|
||||
|
||||
mergecase1
|
||||
mergecase2
|
||||
mergecase3
|
||||
|
||||
resetrc
|
||||
@ -109,8 +109,6 @@ test "x$STORAGE" = 'xtas:_Storage="chunked";'
|
||||
CHUNKSIZES=`cat tmppds.cdl | sed -e '/tas:_ChunkSizes/p' -ed | tr -d '\t \r'`
|
||||
test "x$CHUNKSIZES" = 'xtas:_ChunkSizes=10,15,20;'
|
||||
|
||||
set -x
|
||||
|
||||
echo "*** Test that nccopy -F var1,none works as intended "
|
||||
${NCGEN} -4 -b -o tst_nofilters.nc $srcdir/ref_tst_nofilters.cdl
|
||||
${NCCOPY} -M0 -4 -F var1,none -c // tst_nofilters.nc tmp_nofilters.nc
|
||||
|
||||
39
ncdump/tst_rcmerge.c
Normal file
39
ncdump/tst_rcmerge.c
Normal file
@ -0,0 +1,39 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "netcdf.h"
|
||||
#include "ncrc.h"
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
size_t i,ntriples = 0;
|
||||
NCRCglobalstate* ngs = ncrc_getglobalstate();
|
||||
NCRCinfo* info = NULL;
|
||||
NCTriple* triple = NULL;
|
||||
|
||||
/* Cause the .rc files to be read and merged */
|
||||
nc_initialize();
|
||||
|
||||
if((ngs = ncrc_getglobalstate())==NULL) abort();
|
||||
info = &ngs->rcinfo;
|
||||
|
||||
if(info->ignore) {
|
||||
fprintf(stderr,".rc ignored\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
/* Print out the .rc triples */
|
||||
if((ntriples = NC_rcfile_length(info))==0) {
|
||||
printf("<empty>\n");
|
||||
exit(0);
|
||||
}
|
||||
for(i=0;i<ntriples;i++) {
|
||||
triple = NC_rcfile_ith(info,i);
|
||||
if(triple == NULL) abort();
|
||||
if(triple->host != NULL)
|
||||
printf("[%s] ",triple->host);
|
||||
printf("|%s|->|%s|\n",triple->key,triple->value);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -84,14 +84,19 @@ IF(ENABLE_TESTS)
|
||||
add_sh_test(nczarr_test run_ncgen4)
|
||||
add_sh_test(nczarr_test run_purezarr)
|
||||
add_sh_test(nczarr_test run_interop)
|
||||
add_sh_test(nczarr_test run_misc)
|
||||
|
||||
BUILD_BIN_TEST(tst_chunkcases ${TSTCOMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(tst_chunkcases PUBLIC ../libnczarr)
|
||||
add_sh_test(nczarr_test run_chunkcases)
|
||||
BUILD_BIN_TEST(tst_chunkcases ${TSTCOMMONSRC})
|
||||
TARGET_INCLUDE_DIRECTORIES(tst_chunkcases PUBLIC ../libnczarr)
|
||||
add_sh_test(nczarr_test run_chunkcases)
|
||||
|
||||
if(ENABLE_NCZARR_S3)
|
||||
add_sh_test(nczarr_test run_s3_cleanup)
|
||||
ENDIF()
|
||||
SET_TESTS_PROPERTIES(nczarr_test_run_chunkcases PROPERTIES DEPENDS "nczarr_test_run_misc")
|
||||
SET_TESTS_PROPERTIES(nczarr_test_run_misc PROPERTIES DEPENDS "nczarr_test_run_interop")
|
||||
SET_TESTS_PROPERTIES(nczarr_test_run_interop PROPERTIES DEPENDS "nczarr_test_run_purezarr")
|
||||
|
||||
ENDIF(BUILD_UTILITIES)
|
||||
|
||||
ENDIF(ENABLE_TESTS)
|
||||
|
||||
@ -52,6 +52,12 @@ endif
|
||||
TESTS += run_ncgen4.sh
|
||||
TESTS += run_purezarr.sh
|
||||
TESTS += run_interop.sh
|
||||
TESTS += run_misc.sh
|
||||
|
||||
# Inter-test dependencies
|
||||
run_misc.sh: run_interop.sh
|
||||
run_interop.sh: run_purezarr.sh
|
||||
run_chunkcases.sh: run_misc.sh
|
||||
|
||||
check_PROGRAMS += tst_chunkcases
|
||||
tst_chunkcases_SOURCES = tst_chunkcases.c ${tstcommonsrc}
|
||||
@ -99,7 +105,7 @@ ncdumpchunks_SOURCES = ncdumpchunks.c
|
||||
EXTRA_DIST = CMakeLists.txt \
|
||||
run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \
|
||||
run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \
|
||||
run_purezarr.sh run_interop.sh \
|
||||
run_purezarr.sh run_interop.sh run_misc.sh \
|
||||
ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \
|
||||
ref_ut_map_readmeta.txt ref_ut_map_readmeta2.txt ref_ut_map_search.txt \
|
||||
ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt \
|
||||
@ -110,9 +116,9 @@ ref_perdimspecs.cdl ref_fillonly.cdl \
|
||||
ref_whole.cdl ref_whole.txt \
|
||||
ref_skip.cdl ref_skip.txt ref_skipw.cdl \
|
||||
ref_rem.cdl ref_rem.dmp ref_ndims.cdl ref_ndims.dmp \
|
||||
ref_misc1.cdl ref_misc1.dmp \
|
||||
ref_misc1.cdl ref_misc1.dmp ref_misc2.cdl \
|
||||
ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \
|
||||
ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl
|
||||
ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl
|
||||
|
||||
# Interoperability files
|
||||
EXTRA_DIST += power_901_constants.zip ref_power_901_constants.cdl
|
||||
|
||||
Binary file not shown.
12
nczarr_test/ref_misc2.cdl
Normal file
12
nczarr_test/ref_misc2.cdl
Normal file
@ -0,0 +1,12 @@
|
||||
netcdf tmp_misc2 {
|
||||
dimensions:
|
||||
d0 = 2;
|
||||
d1 = 2;
|
||||
variables:
|
||||
int v(d0, d1) ;
|
||||
data:
|
||||
|
||||
v =
|
||||
0, 1,
|
||||
2, 3 ;
|
||||
}
|
||||
18
nczarr_test/ref_nczarr2zarr.cdl
Normal file
18
nczarr_test/ref_nczarr2zarr.cdl
Normal file
@ -0,0 +1,18 @@
|
||||
netcdf nczarr2zarr {
|
||||
dimensions:
|
||||
.zdim_8 = 8 ;
|
||||
variables:
|
||||
int v(.zdim_8, .zdim_8) ;
|
||||
v:_FillValue = -1 ;
|
||||
data:
|
||||
|
||||
v =
|
||||
_, _, _, _, _, _, _, _,
|
||||
_, _, _, _, _, _, _, _,
|
||||
_, _, _, _, _, _, _, _,
|
||||
_, _, _, _, _, _, _, _,
|
||||
_, _, _, _, 0, 1, 2, 3,
|
||||
_, _, _, _, 4, 5, 6, 7,
|
||||
_, _, _, _, 8, 9, 10, 11,
|
||||
_, _, _, _, 12, 13, 14, 15 ;
|
||||
}
|
||||
@ -3,22 +3,6 @@ dimensions:
|
||||
lat = 361 ;
|
||||
lon = 576 ;
|
||||
variables:
|
||||
float FROCEAN(lat, lon) ;
|
||||
FROCEAN:fmissing_value = 999999986991104. ;
|
||||
FROCEAN:long_name = "fraction_of_ocean" ;
|
||||
FROCEAN:standard_name = "fraction_of_ocean" ;
|
||||
FROCEAN:units = "1" ;
|
||||
FROCEAN:valid_range = -999999986991104., 999999986991104. ;
|
||||
FROCEAN:vmax = 999999986991104. ;
|
||||
FROCEAN:vmin = -999999986991104. ;
|
||||
float FRLAND(lat, lon) ;
|
||||
FRLAND:fmissing_value = 999999986991104. ;
|
||||
FRLAND:long_name = "fraction_of_land" ;
|
||||
FRLAND:standard_name = "fraction_of_land" ;
|
||||
FRLAND:units = "1" ;
|
||||
FRLAND:valid_range = -999999986991104., 999999986991104. ;
|
||||
FRLAND:vmax = 999999986991104. ;
|
||||
FRLAND:vmin = -999999986991104. ;
|
||||
float FRLAKE(lat, lon) ;
|
||||
FRLAKE:fmissing_value = 999999986991104. ;
|
||||
FRLAKE:long_name = "fraction_of_lake" ;
|
||||
@ -27,6 +11,14 @@ variables:
|
||||
FRLAKE:valid_range = -999999986991104., 999999986991104. ;
|
||||
FRLAKE:vmax = 999999986991104. ;
|
||||
FRLAKE:vmin = -999999986991104. ;
|
||||
float FRLAND(lat, lon) ;
|
||||
FRLAND:fmissing_value = 999999986991104. ;
|
||||
FRLAND:long_name = "fraction_of_land" ;
|
||||
FRLAND:standard_name = "fraction_of_land" ;
|
||||
FRLAND:units = "1" ;
|
||||
FRLAND:valid_range = -999999986991104., 999999986991104. ;
|
||||
FRLAND:vmax = 999999986991104. ;
|
||||
FRLAND:vmin = -999999986991104. ;
|
||||
float FRLANDICE(lat, lon) ;
|
||||
FRLANDICE:fmissing_value = 999999986991104. ;
|
||||
FRLANDICE:long_name = "fraction_of_land_ice" ;
|
||||
@ -35,4 +27,12 @@ variables:
|
||||
FRLANDICE:valid_range = -999999986991104., 999999986991104. ;
|
||||
FRLANDICE:vmax = 999999986991104. ;
|
||||
FRLANDICE:vmin = -999999986991104. ;
|
||||
float FROCEAN(lat, lon) ;
|
||||
FROCEAN:fmissing_value = 999999986991104. ;
|
||||
FROCEAN:long_name = "fraction_of_ocean" ;
|
||||
FROCEAN:standard_name = "fraction_of_ocean" ;
|
||||
FROCEAN:units = "1" ;
|
||||
FROCEAN:valid_range = -999999986991104., 999999986991104. ;
|
||||
FROCEAN:vmax = 999999986991104. ;
|
||||
FROCEAN:vmin = -999999986991104. ;
|
||||
}
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
#!/bin/sh
|
||||
|
||||
|
||||
export SETX=1
|
||||
|
||||
# Note that this test builds a special results.<pid> directory in
|
||||
# which to run the tests, where <pid> is the process id number of
|
||||
# the bash shell instance. The reason for doing this is so that
|
||||
@ -13,8 +10,6 @@ export SETX=1
|
||||
# directory for cmake. By running the tests in a separate
|
||||
# results.<pid> I can guarantee that isolation is preserved.
|
||||
|
||||
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ testcasefile() {
|
||||
ref=$1
|
||||
mode=$2
|
||||
if test "x$3" = xmetaonly ; then flags="-h"; fi
|
||||
fileargs ${srcdir}/$ref "mode=$mode,$zext"
|
||||
fileargs ${execdir}/$ref "mode=$mode,$zext"
|
||||
rm -f tmp_${ref}_${zext}.cdl
|
||||
${NCDUMP} $flags $fileurl > tmp_${ref}_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_${ref}.cdl tmp_${ref}_${zext}.cdl
|
||||
@ -27,7 +27,7 @@ testcasezip() {
|
||||
ref=$1
|
||||
mode=$2
|
||||
if test "x$3" = xmetaonly ; then flags="-h"; fi
|
||||
fileargs ${srcdir}/$ref "mode=$mode,$zext"
|
||||
fileargs ${execdir}/$ref "mode=$mode,$zext"
|
||||
rm -f tmp_${ref}_${zext}.cdl
|
||||
${NCDUMP} $flags $fileurl > tmp_${ref}_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_${ref}.cdl tmp_${ref}_${zext}.cdl
|
||||
@ -41,16 +41,20 @@ case "$zext" in
|
||||
rm -fr power_901_constants power_901_constants.file
|
||||
unzip ${srcdir}/power_901_constants.zip > /dev/null
|
||||
mv power_901_constants power_901_constants.file
|
||||
testcasefile power_901_constants xarray metaonly
|
||||
testcasefile power_901_constants zarr metaonly; # test xarray as default
|
||||
;;
|
||||
zip)
|
||||
# Move into position
|
||||
if test "x$srcdir" != "x$execdir" ; then
|
||||
cp ${srcdir}/power_901_constants.zip ${execdir}
|
||||
fi
|
||||
testcasezip power_901_constants xarray metaonly
|
||||
;;
|
||||
*) echo "unimplemented kind: $1" ; exit 1;;
|
||||
esac
|
||||
}
|
||||
|
||||
#testallcases file
|
||||
testallcases file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi
|
||||
#No examples yet: if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi
|
||||
|
||||
|
||||
56
nczarr_test/run_misc.sh
Executable file
56
nczarr_test/run_misc.sh
Executable file
@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
. "$srcdir/test_nczarr.sh"
|
||||
|
||||
# This shell script provides a miscellaneous set of tests
|
||||
|
||||
set -e
|
||||
|
||||
cleanup() {
|
||||
resetrc
|
||||
}
|
||||
|
||||
# Setup the .rc files
|
||||
|
||||
createrc() {
|
||||
RCP="./.ncrc"
|
||||
echo "Creating rc file $RCP"
|
||||
echo "ZARR.DIMENSION_SEPARATOR=/" >>$RCP
|
||||
}
|
||||
|
||||
testcase1() {
|
||||
zext=$1
|
||||
echo "*** Test: use '/' as dimension separator for write then read"
|
||||
fileargs tmp_dimsep "mode=nczarr,$zext"
|
||||
deletemap $zext $file
|
||||
cleanup
|
||||
createrc
|
||||
${NCGEN} -4 -lb -o $fileurl ref_misc1.cdl
|
||||
${NCDUMP} -n tmp_misc1 $fileurl > tmp_misc1_$zext.cdl
|
||||
diff -bw ${srcdir}/ref_misc1.cdl tmp_misc1_$zext.cdl
|
||||
}
|
||||
|
||||
testcase2() {
|
||||
zext=$1
|
||||
echo "*** Test: '/' as dimension separator creates extra groups"
|
||||
fileargs tmp_extra "mode=nczarr,$zext"
|
||||
deletemap $zext $file
|
||||
cleanup
|
||||
createrc
|
||||
${NCGEN} -4 -lb -o "$fileurl" ref_misc2.cdl
|
||||
${NCDUMP} -n tmp_misc2 $fileurl > tmp_misc2_$zext.cdl
|
||||
diff -wb ${srcdir}/ref_misc2.cdl tmp_misc2_$zext.cdl
|
||||
}
|
||||
|
||||
testcase1 file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase1 zip; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testcase1 s3; fi
|
||||
|
||||
testcase2 file
|
||||
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase2 zip; fi
|
||||
if test "x$FEATURE_S3TESTS" = xyes ; then testcase2 s3; fi
|
||||
|
||||
exit 0
|
||||
@ -6,7 +6,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. "$srcdir/test_nczarr.sh"
|
||||
|
||||
# This shell script tests support for:
|
||||
# 1. pure zarr read/write
|
||||
# 1. pure zarr (noxarray) read/write
|
||||
# 2. xarray read/write
|
||||
|
||||
set -e
|
||||
@ -14,19 +14,27 @@ set -e
|
||||
testcase() {
|
||||
zext=$1
|
||||
|
||||
echo "*** Test: pure zarr write; format=$zext"
|
||||
fileargs tmp_purezarr "mode=zarr,$zext"
|
||||
echo "*** Test: pure zarr write then read; format=$zext"
|
||||
fileargs tmp_purezarr "mode=noxarray,$zext"
|
||||
deletemap $zext $file
|
||||
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl
|
||||
${NCDUMP} $fileurl > tmp_purezarr_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_purezarr.cdl tmp_purezarr_${zext}.cdl
|
||||
|
||||
echo "*** Test: xarray zarr write; format=$zext"
|
||||
fileargs tmp_xarray "mode=xarray,$zext"
|
||||
echo "*** Test: xarray zarr write then read; format=$zext"
|
||||
fileargs tmp_xarray "mode=zarr,$zext"
|
||||
deletemap $zext $file
|
||||
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl
|
||||
${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl
|
||||
|
||||
echo "*** Test: pure zarr reading nczarr; format=$zext"
|
||||
fileargs tmp_nczarr "mode=nczarr,$zext"
|
||||
deletemap $zext $file
|
||||
${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl
|
||||
fileargs tmp_nczarr "mode=zarr,$zext"
|
||||
${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl
|
||||
diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl
|
||||
}
|
||||
|
||||
testcase file
|
||||
|
||||
@ -125,3 +125,19 @@ for t in ${TESTS} ; do
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Clear out any existing .rc files
|
||||
WD=`pwd`
|
||||
if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi
|
||||
|
||||
resetrc() {
|
||||
if test "x$RCHOME" = x1 ; then
|
||||
rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc
|
||||
fi
|
||||
rm -f ${WD}/.dodsrc ${WD}/.daprc ${WD}/.ncrc
|
||||
unset NCRCENV_IGNORE
|
||||
unset NCRCENV_RC
|
||||
unset DAPRCFILE
|
||||
}
|
||||
|
||||
resetrc
|
||||
|
||||
@ -107,6 +107,7 @@ done:
|
||||
return THROW(stat);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static void
|
||||
getpathcwd(char** cwdp)
|
||||
{
|
||||
@ -114,15 +115,14 @@ getpathcwd(char** cwdp)
|
||||
(void)NCgetcwd(buf,sizeof(buf));
|
||||
if(cwdp) *cwdp = strdup(buf);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void
|
||||
canonicalfile(char** fp)
|
||||
{
|
||||
size_t len, len2, offset;
|
||||
size_t len;
|
||||
char* f = NULL;
|
||||
char* abspath = NULL;
|
||||
char* p = NULL;
|
||||
char* cwd = NULL;
|
||||
NCURI* uri = NULL;
|
||||
#ifdef _WIN32
|
||||
int fwin32=0, cwd32=0;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user