mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-31 17:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Codify cross-platform file paths

Browse Source 
The netcdf-c code has to deal with a variety of platforms:
Windows, OSX, Linux, Cygwin, MSYS, etc.  These platforms differ
significantly in the kind of file paths that they accept.  So in
order to handle this, I have created a set of replacements for
the most common file system operations such as _open_ or _fopen_
or _access_ to manage the file path differences correctly.

A more limited version of this idea was already implemented via
the ncwinpath.h and dwinpath.c code. So this can be viewed as a
replacement for that code. And in path in many cases, the only
change that was required was to replace '#include <ncwinpath.h>'
with '#include <ncpathmgt.h>' and then replace file operation
calls with the NCxxx equivalent from ncpathmgr.h Note that
recently, the ncwinpath.h was renamed ncpathmgmt.h, so this pull
request should not require dealing with winpath.

The heart of the change is include/ncpathmgmt.h, which provides
alternate operations such as NCfopen or NCaccess and which properly
parse and rebuild path arguments to work for the platform on which
the code is executing. This mostly matters for Windows because of the
way that it uses backslash and drive letters, as compared to *nix*.
One important feature is that the user can do string manipulations
on a file path without having to worry too much about the platform
because the path management code will properly handle most mixed cases.
So one can for example concatenate a path suffix that uses forward
slashes to a Windows path and have it work correctly.

The conversion code is in libdispatch/dpathmgr.c, and the
important function there is NCpathcvt which does the proper
conversions to the local path format.

As a rule, most code should just replace their file operations with
the corresponding NCxxx ones defined in include/ncpathmgmt.h. These
NCxxx functions all call NCpathcvt on their path arguments before
executing the actual file operation.

In some rare cases, the client may need to directly use NCpathcvt,
but this should be avoided as much as possible. If there is a need
for supporting a new file operation not already in ncpathmgmt.h, then
use the code in dpathmgr.c as a template. Also please notify Unidata
so we can include it as a formal part or our supported operations.
Also, if you see an operation in the library that is not using the
NCxxx form, then please submit an issue so we can fix it.

Misc. Changes:
* Clean up the utf8 testing code; it is impossible to get some
  tests to work under windows using shell scripts; the args do
  not pass as utf8 but as some other encoding.
* Added an extra utf8 test case: test_unicode_path.sh
* Add a true test for HDF5 1.10.6 or later because as noted in
  PR https://github.com/Unidata/netcdf-c/pull/1794,
  HDF5 changed its Windows file path handling.
This commit is contained in:
Dennis Heimbigner 2021-03-04 13:41:31 -07:00
parent f388bcec15
commit 0b7a5382e7
55 changed files with 1350 additions and 1304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CMakeLists.txt
View File

@ -662,7 +662,6 @@ IF(USE_HDF5)
IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION)
SET(HDF5_VERSION ${HDF5_VERSION_STRING})
ENDIF()
IF("${HDF5_VERSION}" STREQUAL "")
MESSAGE(STATUS "Unable to determine hdf5 version. NetCDF requires at least version ${HDF5_VERSION_REQUIRED}")
ELSE()
@ -803,6 +802,15 @@ IF(USE_HDF5)
SET(HAS_PAR_FILTERS no CACHE STRING "")
ENDIF()
# Check to see if HDF5 library is 1.10.6 or greater.
# Used to control path name conversion
IF(${HDF5_VERSION} VERSION_LESS "1.10.6")
SET(HDF5_UTF8_PATHS FALSE)
ELSE()
SET(HDF5_UTF8_PATHS TRUE)
ENDIF()
MESSAGE("-- Checking for HDF5 version 1.10.6 or later: ${HDF5_UTF8_PATHS}")
SET(H5_USE_16_API 1)
OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
IF(NOT NC_ENABLE_HDF_16_API)
@ -2204,6 +2212,7 @@ configure_file(
SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in)
SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
SET(ISMSVC "${MSVC}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)

4
configure.ac
View File

@ -79,9 +79,9 @@ AC_MSG_NOTICE([checking supported formats])
# An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
[(synonym for --enable-hdf5)])])
[(deprecated synonym for --enable-hdf5)])])
test "x$enable_netcdf4" = xno || enable_netcdf4=yes
AC_MSG_RESULT([$enable_netcdf4])
AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)])
AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)])
AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
[(synonym for --disable-netcdf4)])])

2
debug/cf.cmake
View File

@ -15,7 +15,7 @@ S3TEST=1
#CDF5=1
#HDF4=1
#TR=--trace
H518=1
for arg in "$@" ; do
case "$arg" in

2
include/Makefile.am
View File

@ -19,7 +19,7 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h ncfilter.h
if USE_DAP
noinst_HEADERS += ncdap.h

20
include/hdf5internal.h
View File

@ -201,23 +201,9 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd
extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
#ifdef _WIN32
extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* Maxinum length of a typical path in UTF-8.
* When converting from ANSI to UTF-8, the length will be up to 3 times,
* so round up 260*3 to 1024. (260=MAX_PATH) */
#define MAX_PATHBUF_SIZE 1024
/* Struct for converting ANSI to UTF-8. */
typedef struct pathbuf
{
void *ptr;
char buffer[MAX_PATHBUF_SIZE];
} pathbuf_t;
const char *nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path);
void nc4_hdf5_free_pathbuf(pathbuf_t *pb);
#endif /* _WIN32 */
EXTERNL hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
EXTERNL hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);
#endif /* _HDF5INTERNAL_ */

1
include/nchttp.h
View File

@ -20,6 +20,7 @@ typedef struct NC_HTTP_STATE {
const char** headset; /* which headers to capture */
NClist* headers;
NCbytes* buf;
char errbuf[1024]; /* assert(CURL_ERROR_SIZE <= 1024) */
} NC_HTTP_STATE;
extern int nc_http_open(const char* objecturl, NC_HTTP_STATE** state, size64_t* lenp);

97
include/ncpathmgr.h
View File

@ -2,8 +2,8 @@
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#ifndef _NCWINIO_H_
#define _NCWINIO_H_
#ifndef _NCPATHMGR_H_
#define _NCPATHMGR_H_
#include "config.h"
#include <stdlib.h>
@ -28,6 +28,7 @@
#endif
#endif
/* Define wrapper constants for use with NCaccess */
/* Define wrapper constants for use with NCaccess */
#ifdef _WIN32
#define ACCESS_MODE_EXISTS 0
@ -49,16 +50,63 @@
#define ACCESS_MODE_RW (R_OK|W_OK)
#endif
/* Path Converter */
#ifdef _WIN32
#ifndef S_IFDIR
#define S_IFDIR _S_IFDIR
#define S_IFREG _S_IFREG
#endif
#ifndef S_ISDIR
#define S_ISDIR(mode) ((mode) & _S_IFDIR)
#define S_ISREG(mode) ((mode) & _S_IFREG)
#endif
#endif /*_WIN32*/
/*
WARNING: you should never need to explictly call this function;
rather it is invoked as part of the wrappers for e.g. NCfopen, etc.
This function attempts to take an arbitrary path and convert
it to a canonical form.
Assumptions about Input path:
1. It is a relative or absolute path
2. It is not a URL
3. It conforms to the format expected by one of the following:
Linux (/x/y/...), Cygwin (/cygdrive/D/...),
Windows (D:/...), or MSYS (/D/...), or relative (x/y...)
4. It is encoded in the local platform character set.
Note that for most systems, this is utf-8. But for Windows,
the encoding is most likely some form of ANSI code page, probably
the windows 1252 encoding.
Note that in any case, the path must be representable in the
local Code Page.
On output it produces a re-written path that has the following
properties:
1. The path is normalized to match the platform on which the code
is running (e.g. cygwin, windows, msys, linux). So for example
using a cygwin path under visual studio will convert e.g.
/cygdrive/d/x/y to d:\x\y. See ../unit_test/test_pathcvt.c
for example conversions.
It returns the converted path.
Note that this function is intended to be Idempotent: f(f(x) == f(x).
This means it is ok to call it repeatedly with no harm.
*/
EXTERNL char* NCpathcvt(const char* path);
/* path -> URL Path converter */
EXTERNL char* NCurlpath(const char* path);
/* Canonicalize and make absolute by prefixing the current working directory */
EXTERNL char* NCpathabsolute(const char* name);
/* Fix path in case it was escaped by shell */
EXTERNL char* NCdeescape(const char* name);
/* Convert from the local coding (e.g. ANSI) to utf-8;
note that this can produce unexpected results for Windows
because it first converts to wide character and then to utf8. */
EXTERNL int NCpath2utf8(const char* path, char** u8p);
#ifdef WINPATH
/* Wrap various stdio and unistd IO functions.
It is especially important to use for windows so that
NCpathcvt (above) is invoked on the path.
*/
#if defined(WINPATH)
/* path converter wrappers*/
EXTERNL FILE* NCfopen(const char* path, const char* flags);
EXTERNL int NCopen3(const char* path, int flags, int mode);
@ -68,6 +116,10 @@ EXTERNL int NCremove(const char* path);
EXTERNL int NCmkdir(const char* path, int mode);
EXTERNL int NCrmdir(const char* path);
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
#ifdef HAVE_SYS_STAT_H
EXTERNL int NCstat(char* path, struct stat* buf);
#endif
#ifdef HAVE_DIRENT_H
EXTERNL DIR* NCopendir(const char* path);
EXTERNL int NCclosedir(DIR* ent);
@ -78,8 +130,12 @@ EXTERNL int NCclosedir(DIR* ent);
#define NCopen2(path,flags) open((path),(flags))
#define NCremove(path) remove(path)
#define NCaccess(path,mode) access(path,mode)
#define NCmkdir(path,mode) mkdir(path,mode)
#define NCgetcwd(buf,len) getcwd(buf,len)
#ifdef HAVE_SYS_STAT_H
#define NCstat(path,buf) stat(path,buf)
#endif
#define NCcwd(buf, len) getcwd(buf,len)
#define NCmkdir(path, mode) mkdir(path,mode)
#define NCrmdir(path) rmdir(path)
#ifdef HAVE_DIRENT_H
#define NCopendir(path) opendir(path)
@ -89,7 +145,26 @@ EXTERNL int NCclosedir(DIR* ent);
/* Platform independent */
#define NCclose(fd) close(fd)
#define NCfstat(fd,buf) fstat(fd,buf)
EXTERNL int NChasdriveletter(const char* path);
/**************************************************/
/* Following definitions are for testing only */
#endif /* _NCWINIO_H_ */
/* Possible Kinds Of Output */
#define NCPD_UNKNOWN 0
#define NCPD_NIX 1
#define NCPD_MSYS 2
#define NCPD_CYGWIN 3
#define NCPD_WIN 4
#define NCPD_REL 5 /* actual kind is unknown */
EXTERNL char* NCpathcvt_test(const char* path, int ukind, int udrive);
EXTERNL void printutf8hex(const char* s, char* sx);
/**************************************************/
/* From dutil.c */
EXTERNL char* NC_backslashEscape(const char* s);
EXTERNL char* NC_backslashUnescape(const char* esc);
#endif /* _NCPATHMGR_H_ */

2
include/ncrc.h
View File

@ -55,8 +55,6 @@ extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
/* From dutil.c (Might later move to e.g. nc.h */
extern int NC__testurl(const char* path, char** basenamep);
extern int NC_isLittleEndian(void);
extern char* NC_backslashEscape(const char* s);
extern char* NC_backslashUnescape(const char* esc);
extern char* NC_entityescape(const char* s);
extern int NC_readfile(const char* filename, NCbytes* content);
extern int NC_writefile(const char* filename, size_t size, void* content);

3
include/ncuri.h
View File

@ -67,6 +67,9 @@ EXTERNL void ncurifree(NCURI* ncuri);
/* Replace the protocol */
EXTERNL int ncurisetprotocol(NCURI*,const char* newprotocol);
/* Replace the path */
EXTERNL int ncurisetpath(NCURI*,const char* newpath);
/* Replace the constraints */
EXTERNL int ncurisetquery(NCURI*,const char* query);

2
lib_flags.am
View File

@ -13,7 +13,7 @@ if USE_DAP
AM_CPPFLAGS += -I${top_srcdir}/oc2
endif
if HAVE_AWS
if ENABLE_S3_SDK
AM_LDFLAGS += -lstdc++
endif

45
libdispatch/ddispatch.c
View File

@ -10,6 +10,7 @@ See LICENSE.txt for license information.
#include "ncbytes.h"
#include "ncrc.h"
#include "ncoffsets.h"
#include "ncpathmgr.h"
/* Required for getcwd, other functions. */
#ifdef HAVE_UNISTD_H
@ -19,7 +20,6 @@ See LICENSE.txt for license information.
/* Required for getcwd, other functions. */
#ifdef _WIN32
#include <direct.h>
#define getcwd _getcwd
#endif
#if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
@ -57,61 +57,28 @@ NCDISPATCH_initialize(void)
/* Capture temp dir*/
{
char* tempdir;
char* p;
char* q;
char cwd[4096];
#ifdef _WIN32
char* tempdir = NULL;
#if defined _WIN32 || defined __MSYS__
tempdir = getenv("TEMP");
#else
tempdir = "/tmp";
#endif
if(tempdir == NULL) {
fprintf(stderr,"Cannot find a temp dir; using ./\n");
tempdir = getcwd(cwd,sizeof(cwd));
if(tempdir == NULL || *tempdir == '\0') tempdir = ".";
tempdir = ".";
}
globalstate->tempdir= (char*)malloc(strlen(tempdir) + 1);
for(p=tempdir,q=globalstate->tempdir;*p;p++,q++) {
if((*p == '/' && *(p+1) == '/')
|| (*p == '\\' && *(p+1) == '\\')) {p++;}
*q = *p;
}
*q = '\0';
#ifdef _WIN32
#else
/* Canonicalize */
for(p=globalstate->tempdir;*p;p++) {
if(*p == '\\') {*p = '/'; };
}
*q = '\0';
#endif
globalstate->tempdir= strdup(tempdir);
}
/* Capture $HOME */
{
char* p;
char* q;
char* home = getenv("HOME");
if(home == NULL) {
/* use tempdir */
home = globalstate->tempdir;
}
globalstate->home = (char*)malloc(strlen(home) + 1);
for(p=home,q=globalstate->home;*p;p++,q++) {
if((*p == '/' && *(p+1) == '/')
|| (*p == '\\' && *(p+1) == '\\')) {p++;}
*q = *p;
}
*q = '\0';
#ifdef _WIN32
#else
/* Canonicalize */
for(p=home;*p;p++) {
if(*p == '\\') {*p = '/'; };
}
#endif
globalstate->home = strdup(home);
}
/* Now load RC File */

10
libdispatch/dfile.c
View File

@ -1853,12 +1853,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
/* Skip past any leading whitespace in path */
const unsigned char* p;
for(p=(const unsigned char*)path0;*p;p++) {if(*p > ' ') break;}
#ifdef WINPATH
/* Need to do path conversion */
path = NCpathcvt((const char*)p);
#else
path = nulldup((const char*)p);
#endif
}
memset(&model,0,sizeof(model));
@ -2009,12 +2004,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
/* Skip past any leading whitespace in path */
const char* p;
for(p=(const char*)path0;*p;p++) {if(*p < 0 || *p > ' ') break;}
#ifdef WINPATH
/* Need to do path conversion */
path = NCpathcvt(p);
#else
path = nulldup(p);
#endif
}
memset(&model,0,sizeof(model));

32
libdispatch/dhttp.c
View File

@ -24,9 +24,10 @@
#include "nclist.h"
#include "nchttp.h"
#undef VERBOSE
#undef TRACE
#define CURLERR(e) (e)
#define CURLERR(e) reporterror(state,(e))
/* Mnemonics */
#define GETCMD 0
@ -40,6 +41,8 @@ static int setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf);
static int execute(NC_HTTP_STATE* state, int headcmd);
static int headerson(NC_HTTP_STATE* state, const char** which);
static void headersoff(NC_HTTP_STATE* state);
static void showerrors(NC_HTTP_STATE* state);
static int reporterror(NC_HTTP_STATE* state, CURLcode cstat);
#ifdef TRACE
static void
@ -81,6 +84,15 @@ nc_http_open(const char* objecturl, NC_HTTP_STATE** statep, size64_t* filelenp)
/* initialize curl*/
state->curl = curl_easy_init();
if (state->curl == NULL) {stat = NC_ECURL; goto done;}
showerrors(state);
#ifdef VERBOSE
{long onoff = 1;
CURLcode cstat = CURLE_OK;
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_VERBOSE, onoff));
if(cstat != CURLE_OK)
{stat = NC_ECURL; goto done;}
}
#endif
if(filelenp) {
*filelenp = -1;
/* Attempt to get the file length using HEAD */
@ -324,6 +336,9 @@ setupconn(NC_HTTP_STATE* state, const char* objecturl, NCbytes* buf)
if(objecturl != NULL) {
/* Set the URL */
#ifdef TRACE
fprintf(stderr,"curl.setup: url |%s|\n",objecturl);
#endif
cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_URL, (void*)objecturl));
if (cstat != CURLE_OK) goto fail;
}
@ -419,3 +434,18 @@ headersoff(NC_HTTP_STATE* state)
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, NULL));
(void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, NULL));
}
static void
showerrors(NC_HTTP_STATE* state)
{
(void)curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errbuf);
}
static int
reporterror(NC_HTTP_STATE* state, CURLcode cstat)
{
if(cstat != CURLE_OK)
fprintf(stderr,"curlcode: (%d)%s : %s\n",
cstat,curl_easy_strerror(cstat),state->errbuf);
return cstat;
}

28
libdispatch/dinfermodel.c
View File

@ -860,34 +860,6 @@ nullify(const char* s)
#endif
/**************************************************/
#if 0
/* return 1 if path looks like a url; 0 otherwise */
int
NC_testurl(const char* path)
{
int isurl = 0;
NCURI* tmpurl = NULL;
if(path == NULL) return 0;
/* Ok, try to parse as a url */
if(ncuriparse(path,&tmpurl)==NCU_OK) {
/* Do some extra testing to make sure this really is a url */
/* Look for a known/accepted protocol */
struct NCPROTOCOLLIST* protolist;
for(protolist=ncprotolist;protolist->protocol;protolist++) {
if(strcmp(tmpurl->protocol,protolist->protocol) == 0) {
isurl=1;
break;
}
}
ncurifree(tmpurl);
return isurl;
}
return 0;
}
#endif
/**************************************************/
/**
* Provide a hidden interface to allow utilities

965
libdispatch/dpathmgr.c
View File

File diff suppressed because it is too large Load Diff

2
libdispatch/drc.c
View File

@ -433,7 +433,7 @@ rcsearch(const char* prefix, const char* rcname, char** pathp)
strncat(path,"/",pathlen);
strncat(path,rcname,pathlen);
/* see if file is readable */
f = fopen(path,"r");
f = NCfopen(path,"r");
if(f != NULL)
nclog(NCLOGDBG, "Found rc file=%s",path);
done:

5
libdispatch/dutil.c
View File

@ -182,7 +182,8 @@ NC_mktmp(const char* base)
mode_t mask;
#endif
/* Make sure that this path conversion has been applied */
/* Make sure that this path conversion has been applied
since we do not wrap mkstemp */
cvtpath = NCpathcvt(base);
strncpy(tmp,cvtpath,sizeof(tmp));
nullfree(cvtpath);
@ -214,7 +215,7 @@ NC_mktmp(const char* base)
strncat(tmp,spid,sizeof(tmp) - strlen(tmp) - 1);
}
#endif /* HAVE_MKTEMP */
#ifdef _MSC_VER
#ifdef _WIN32
fd=NCopen3(tmp,O_RDWR|O_BINARY|O_CREAT, _S_IREAD|_S_IWRITE);
#else
fd=NCopen3(tmp,O_RDWR|O_CREAT|O_EXCL, S_IRWXU);

9
libdispatch/ncuri.c
View File

@ -485,6 +485,15 @@ ncurisetprotocol(NCURI* duri,const char* protocol)
return (NC_NOERR);
}
/* Replace the path */
int
ncurisetpath(NCURI* duri,const char* newpath)
{
nullfree(duri->path);
duri->path = strdup(newpath);
return (NC_NOERR);
}
/* Replace the query */
int
ncurisetquery(NCURI* duri,const char* query)

39
libhdf5/hdf5create.c
View File

@ -9,6 +9,9 @@
*/
#include "config.h"
#include "netcdf.h"
#include "ncpathmgr.h"
#include "ncpathmgr.h"
#include "hdf5internal.h"
/* From hdf5file.c. */
@ -22,12 +25,6 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_C
/* From nc4mem.c */
extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t);
#ifdef _WIN32
static hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);
#else
#define nc4_H5Fcreate H5Fcreate
#endif
/**
* @internal Create a netCDF-4/HDF5 file.
*
@ -114,7 +111,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
/* If this file already exists, and NC_NOCLOBBER is specified,
return an error (unless diskless|inmemory) */
if (!nc4_info->mem.diskless && !nc4_info->mem.inmemory) {
if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) {
if ((cmode & NC_NOCLOBBER) && (fp = NCfopen(path, "r"))) {
fclose(fp);
BAIL(NC_EEXIST);
}
@ -313,8 +310,6 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
return res;
}
#ifdef _WIN32
/**
* Wrapper function for H5Fcreate.
* Converts the filename from ANSI to UTF-8 as needed before calling H5Fcreate.
@ -325,18 +320,24 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
* @param fapl_id File access property list identifier.
* @return A file identifier if succeeded. A negative value if failed.
*/
static hid_t
nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
hid_t
nc4_H5Fcreate(const char *filename0, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
{
pathbuf_t pb;
hid_t hid;
char* localname = NULL;
char* filename = NULL;
filename = nc4_ndf5_ansi_to_utf8(&pb, filename);
if (!filename)
return H5I_INVALID_HID;
hid = H5Fcreate(filename, flags, fcpl_id, fapl_id);
nc4_hdf5_free_pathbuf(&pb);
#ifdef HDF5_UTF8_PATHS
NCpath2utf8(filename0,&filename);
#else
filename = strdup(filename0);
#endif
/* Canonicalize it since we are not opening the file ourselves */
if((localname = NCpathcvt(filename))==NULL)
{hid = H5I_INVALID_HID; goto done;}
hid = H5Fcreate(localname, flags, fcpl_id, fapl_id);
done:
nullfree(filename);
nullfree(localname);
return hid;
}
#endif /* _WIN32 */

3
libhdf5/hdf5internal.c
View File

@ -1026,7 +1026,7 @@ nc_log_hdf5(void)
#endif /* LOGGING */
#if 0
#ifdef _WIN32
/**
@ -1112,3 +1112,4 @@ nc4_hdf5_free_pathbuf(pathbuf_t *pb)
}
#endif /* _WIN32 */
#endif /*0*/

39
libhdf5/hdf5open.c
View File

@ -15,6 +15,8 @@
#include "ncrc.h"
#include "ncauth.h"
#include "ncmodel.h"
#include "ncfilter.h"
#include "ncpathmgr.h"
#ifdef ENABLE_BYTERANGE
#include "H5FDhttp.h"
@ -67,15 +69,11 @@ extern int NC4_open_image_file(NC_FILE_INFO_T* h5);
/* Defined later in this file. */
static int rec_read_metadata(NC_GRP_INFO_T *grp);
#ifdef _WIN32
static hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
#else
#define nc4_H5Fopen H5Fopen
#endif
#ifdef ENABLE_BYTERANGE
#ifdef ENABLE_HDF5_ROS3
static int ros3info(NCauth** auth, NCURI* uri, char** hostportp, char** regionp);
#endif
#endif
/**
* @internal Struct to track HDF5 object info, for
@ -2759,6 +2757,7 @@ exit:
return retval;
}
#ifdef ENABLE_BYTERANGE
#ifdef ENABLE_HDF5_ROS3
static int
ros3info(NCauth** authp, NCURI* uri, char** hostportp, char** regionp)
@ -2806,8 +2805,7 @@ done:
return stat;
}
#endif /*ENABLE_HDF5_ROS3*/
#ifdef _WIN32
#endif /*ENABLE_BYTERANGE*/
/**
* Wrapper function for H5Fopen.
@ -2818,18 +2816,23 @@ done:
* @param fapl_id File access property list identifier.
* @return A file identifier if succeeded. A negative value if failed.
*/
static hid_t
nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id)
hid_t
nc4_H5Fopen(const char *filename0, unsigned flags, hid_t fapl_id)
{
pathbuf_t pb;
hid_t hid;
char* localname = NULL;
char* filename = NULL;
filename = nc4_ndf5_ansi_to_utf8(&pb, filename);
if (!filename)
return H5I_INVALID_HID;
hid = H5Fopen(filename, flags, fapl_id);
nc4_hdf5_free_pathbuf(&pb);
#ifdef HDF5_UTF8_PATHS
NCpath2utf8(filename0,&filename);
#else
filename = strdup(filename0);
#endif
if((localname = NCpathcvt(filename))==NULL)
{hid = H5I_INVALID_HID; goto done;}
hid = H5Fopen(localname, flags, fapl_id);
done:
nullfree(filename);
nullfree(localname);
return hid;
}
#endif /* _WIN32 */

5
libhdf5/nc4memcb.c
View File

@ -83,6 +83,7 @@
#include <hdf5_hl.h>
#include "nc4internal.h"
#include "hdf5internal.h"
#ifndef HDrealloc
#define HDrealloc(x,y) realloc(x,y)
@ -806,10 +807,10 @@ NC4_image_init(NC_FILE_INFO_T* h5)
/* Assign file image in FAPL to the core file driver */
if(create) {
if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0)
if ((file_id = nc4_H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0)
goto out;
} else {
if ((file_id = H5Fopen(file_name, file_open_flags, fapl)) < 0)
if ((file_id = nc4_H5Fopen(file_name, file_open_flags, fapl)) < 0)
goto out;
}

1
libnczarr/zmap.h
View File

@ -356,7 +356,6 @@ EXTERNL void nczm_sortlist(struct NClist* l);
EXTERNL void nczm_sortenvv(int n, char** envv);
EXTERNL void NCZ_freeenvv(int n, char** envv);
#ifdef __cplusplus
}
#endif

6
libsrc/ncFile.c
View File

@ -7,6 +7,8 @@
#include <stdio.h>
#include <errno.h>
#include "ncpathmgr.h"
/*
Implement the ncstdio.h interface using unix stdio.h
*/
@ -84,9 +86,9 @@ ncFile_open(const char *path, int ioflags, ncstdio** filepp)
File* f;
if(fIsSet(ioflags,NC_NOCLOBBER))
f = fopen(path,"r");
f = NCfopen(path,"r");
else
f = fopen(path,"w+");
f = NCfopen(path,"w+");
if(f == NULL)
return errno;

9
libsrc/posixio.c
View File

@ -47,6 +47,7 @@
#define SEEK_END 2
#endif
#include "ncpathmgr.h"
#include "ncio.h"
#include "fbits.h"
#include "rnd.h"
@ -1622,10 +1623,10 @@ posixio_create(const char *path, int ioflags,
fSet(oflags, O_BINARY);
#endif
#ifdef vms
fd = open(path, oflags, NC_DEFAULT_CREAT_MODE, "ctx=stm");
fd = NCopen3(path, oflags, NC_DEFAULT_CREAT_MODE, "ctx=stm");
#else
/* Should we mess with the mode based on NC_SHARE ?? */
fd = open(path, oflags, NC_DEFAULT_CREAT_MODE);
fd = NCopen3(path, oflags, NC_DEFAULT_CREAT_MODE);
#endif
#if 0
(void) fprintf(stderr, "ncio_create(): path=\"%s\"\n", path);
@ -1760,9 +1761,9 @@ posixio_open(const char *path,
#endif
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
fd = NCopen3(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, 0);
fd = NCopen3(path, oflags, 0);
#endif
if(fd < 0)
{

9
libsrc/winceio.c
View File

@ -31,6 +31,7 @@
#include <string.h>
#include <stdio.h>
#include "ncpathmgr.h"
#include "ncio.h"
#include "fbits.h"
#include "rnd.h"
@ -508,9 +509,9 @@ ncio_create(const char *path, int ioflags,
/* Since we do not have use of the O_EXCL flag,
we need to fake it */
#ifdef WINCE
f = fopen(path,"rb");
f = NCfopen(path,"rb");
#else
f = fopen(path,"r");
f = NCfopen(path,"r");
#endif
if(f != NULL) { /* do not overwrite */
(void)fclose(f);
@ -518,7 +519,7 @@ ncio_create(const char *path, int ioflags,
}
}
f = fopen(path, oflags);
f = NCfopen(path, oflags);
if(f == NULL)
{
status = errno;
@ -602,7 +603,7 @@ ncio_open(const char *path,
if(nciop == NULL)
return ENOMEM;
f = fopen(path, oflags);
f = NCfopen(path, oflags);
if(f == NULL)
{

6
nc_test/CMakeLists.txt
View File

@ -36,7 +36,11 @@ TARGET_LINK_LIBRARIES(nc_test
)
# Some extra stand-alone tests
SET(TESTS t_nc tst_small tst_misc tst_norm tst_names tst_nofill tst_nofill2 tst_nofill3 tst_meta tst_inq_type tst_utf8_validate tst_utf8_phrases tst_global_fillval tst_max_var_dims tst_formats tst_def_var_fill tst_err_enddef tst_default_format)
SET(TESTS t_nc tst_small tst_misc tst_norm tst_names tst_nofill tst_nofill2 tst_nofill3 tst_meta tst_inq_type tst_utf8_phrases tst_global_fillval tst_max_var_dims tst_formats tst_def_var_fill tst_err_enddef tst_default_format)
IF(NOT MSVC)
SET(TESTS ${TESTS} tst_utf8_validate)
ENDIF()
IF(NOT HAVE_BASH)
SET(TESTS ${TESTS} tst_atts3)

2
nc_test/Makefile.am
View File

@ -8,7 +8,9 @@
# Un comment to use a more verbose test driver
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT = export SETX=1;
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am

26
ncdump/CMakeLists.txt
View File

@ -94,7 +94,6 @@ IF(MSVC)
ENDIF()
IF(ENABLE_TESTS)
ADD_EXECUTABLE(rewrite-scalar rewrite-scalar.c)
ADD_EXECUTABLE(bom bom.c)
@ -146,8 +145,8 @@ IF(ENABLE_TESTS)
SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
ENDIF(USE_HDF5)
ENDIF(MSVC)
ENDIF(USE_HDF5)
ENDIF(MSVC)
# Base tests
# The tests are set up as a combination of shell scripts and executables that
@ -169,7 +168,6 @@ ENDIF(MSVC)
add_sh_test(ncdump tst_fileinfo)
add_sh_test(ncdump tst_hdf5_offset)
ENDIF(USE_HDF5)
add_sh_test(ncdump test_unicode_directory)
add_sh_test(ncdump tst_null_byte_padding)
IF(USE_STRICT_NULL_BYTE_HEADER_PADDING)
@ -240,8 +238,6 @@ ENDIF(MSVC)
build_bin_test_no_prefix(tst_h_rdc0)
ENDIF()
build_bin_test_no_prefix(tst_unicode)
build_bin_test_no_prefix(tst_fillbug)
add_sh_test(ncdump_sh tst_fillbug)
@ -277,6 +273,8 @@ ENDIF(MSVC)
add_sh_test(ncdump tst_ncgen4)
ENDIF(USE_HDF5)
ENDIF(USE_HDF5)
add_sh_test(ncdump tst_inttags)
IF(USE_HDF5)
add_sh_test(ncdump tst_inttags4)
@ -291,13 +289,23 @@ ENDIF(MSVC)
PROPERTIES ENVIRONMENT NC_VLEN_NOTEST=1)
ENDIF()
# The unicode tests are complicated
IF(USE_HDF5)
IF(HAVE_BASH)
build_bin_test_no_prefix(tst_unicode)
IF(NOT MSVC)
# These tests do not work under windows
add_sh_test(ncdump test_unicode_directory)
add_sh_test(ncdump test_unicode_path)
ENDIF(NOT MSVC)
ENDIF(HAVE_BASH)
ENDIF(USE_HDF5)
IF(USE_CDF5)
add_sh_test(ncdump test_keywords)
ENDIF()
ENDIF()
ENDIF()
ENDIF(ENABLE_TESTS)
#IF(MSVC)
# SET_TARGET_PROPERTIES(ncdump

3
ncdump/Makefile.am
View File

@ -162,7 +162,8 @@ ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \
ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \
ref_no_ncproperty.nc test_unicode_directory.sh \
ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \
test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_tst_nofilters.cdl
test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_tst_nofilters.cdl \
test_unicode_path.sh
# The L512.bin file is file containing exactly 512 bytes each of value 0.
# It is used for creating hdf5 files with varying offsets for testing.

1
ncdump/nc4print.c
View File

@ -16,6 +16,7 @@ create other tools.
#include <stdio.h>
#include "netcdf.h"
#include "ncbytes.h"
#include "ncpathmgr.h"
extern int NC4print(NCbytes* buf, int ncid);

8
ncdump/nccopy.c
View File

@ -32,6 +32,7 @@
#include "nccomps.h"
#include "list.h"
#include "ncpathmgr.h"
#include "ncfilter.h"
#undef DEBUGFILTER
#undef DEBUGCHUNK
@ -2410,10 +2411,9 @@ main(int argc, char**argv)
if (argc != 2) {
error("one input file and one output file required");
}
/* Escape the file names in case bash escaped them */
inputfile = NCdeescape(argv[0]);
outputfile = NCdeescape(argv[1]);
/* Canonicalize the input and output files names */
inputfile = NC_backslashUnescape(argv[0]); /* Remove shell added escapes */
outputfile = NC_backslashUnescape(argv[1]);
if(strcmp(inputfile, outputfile) == 0) {
error("output would overwrite input");
}

64
ncdump/ncdump.c
View File

@ -25,9 +25,6 @@ Research/Unidata. See \ref copyright file for more info. */
#include <ctype.h>
#include <assert.h>
#include <math.h>
#ifdef HAVE_LOCALE_H
#include <locale.h>
#endif /* HAVE_LOCALE_H */
#include "netcdf.h"
#include "netcdf_mem.h"
@ -47,6 +44,7 @@ Research/Unidata. See \ref copyright file for more info. */
#include "nclist.h"
#include "ncuri.h"
#include "nc_provenance.h"
#include "ncpathmgr.h"
#ifdef USE_NETCDF4
#include "nc4internal.h" /* to get name of the special properties file */
@ -145,39 +143,35 @@ usage(void)
static char *
name_path(const char *path)
{
const char *cp;
char *new;
char *sp;
char* cvtpath = NULL;
const char *cp = NULL;
char *sp = NULL;
size_t cplen = 0;
char* base = NULL;
#ifdef vms
#define FILE_DELIMITER ']'
#endif
#if defined(_WIN32) || defined(msdos)
#define FILE_DELIMITER '\\'
#endif
#ifndef FILE_DELIMITER /* default to unix */
#define FILE_DELIMITER '/'
#endif
if((cvtpath = NCpathcvt(path))==NULL)
return NULL;
/* See if this is a url */
{
char* base;
if(nc__testurl(path,&base)) {
return base; /* Looks like a url */
}
/* else fall thru and treat like a file path */
}
if(nc__testurl(cvtpath,&base))
goto done; /* Looks like a url */
/* else fall thru and treat like a file path */
cp = strrchr(path, FILE_DELIMITER);
if (cp == 0) /* no delimiter */
cp = path;
cp = strrchr(cvtpath, '/');
if (cp == NULL) /* no delimiter */
cp = cvtpath;
else /* skip delimiter */
cp++;
new = (char *) emalloc((unsigned) (strlen(cp)+1));
(void) strncpy(new, cp, strlen(cp) + 1); /* copy last component of path */
if ((sp = strrchr(new, '.')) != NULL)
cplen = strlen(cp);
base = (char *) emalloc((unsigned) (cplen+1));
base[0] = '\0';
strlcat(base,cp,cplen+1);
if ((sp = strrchr(base, '.')) != NULL)
*sp = '\0'; /* strip off any extension */
return new;
done:
nullfree(cvtpath);
return base;
}
/* Return primitive type name */
@ -333,7 +327,7 @@ fileopen(const char* path, void** memp, size_t* sizep)
#endif
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
fd = NCopen3(path, oflags, 0, "ctx=stm");
#else
fd = NCopen2(path, oflags);
#endif
@ -2182,9 +2176,6 @@ main(int argc, char *argv[])
putenv("PRINTF_EXPONENT_DIGITS=2"); /* Enforce unix/linux style exponent formatting. */
#endif
#ifdef HAVE_LOCALE_H
setlocale(LC_ALL, "C"); /* CDL may be ambiguous with other locales */
#endif /* HAVE_LOCALE_H */
progname = argv[0];
set_formats(FLT_DIGITS, DBL_DIGITS); /* default for float, double data */
@ -2342,10 +2333,11 @@ main(int argc, char *argv[])
init_epsilons();
/* Deescape the user name */
path = NCdeescape(argv[i]);
/* We need to look for escape characters because the argument
may have come in via a shell script */
path = NC_backslashUnescape(argv[i]);
if(path == NULL) {
snprintf(errmsg,sizeof(errmsg),"out of memory copying argument %s", argv[i]);
snprintf(errmsg,sizeof(errmsg),"out of memory un-escaping argument %s", argv[i]);
goto fail;
}

1
ncdump/ocprint.c
View File

@ -26,6 +26,7 @@
#include "ncuri.h"
#include "ncbytes.h"
#include "nclog.h"
#include "ncpathmgr.h"
#include "oc.h"
#include "ocx.h"

26
ncdump/test_unicode_directory.sh
View File

@ -8,7 +8,6 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
ERR() {
RES=$?
if [ $RES -ne 0 ]; then
@ -17,21 +16,28 @@ ERR() {
fi
}
LC_ALL="C.UTF-8"
export LC_ALL
#UNISTRING='海'
UNISTRING=$(echo '\xe6\xb5\xb7')
#UNISTRING=$(echo '\xe6\xb5\xb7')
UNISTRING='海'
echo ""
echo "Creating Unicode String Directory ${UNISTRING}"
mkdir -p ${UNISTRING}; ERR
mkdir -p "${UNISTRING}"; ERR
ls -ld "${UNISTRING}"
echo "*** Generating binary file ${UNISTRING}/tst_utf.nc..."
# Do test for netcdf-3 and (optionally) netcdf-4
echo "*** Generating netcdf-3 binary file ${UNISTRING}/tst_utf.nc..."
${NCGEN} -b -o "${UNISTRING}/tst_utf.nc" "${srcdir}/ref_tst_utf8.cdl"; ERR
echo "*** Accessing binary file ${UNISTRING}/tst_utf.nc..."
${NCDUMP} -h "${UNISTRING}/tst_utf.nc"; ERR
if test "x$FEATURE_HDF5" = xyes ; then
echo "*** Generating netcdf-4 binary file ${UNISTRING}/tst_utf.nc..."
rm -f "${UNISTRING}/tst_utf.nc"
${NCGEN} -4 -b -o "${UNISTRING}/tst_utf.nc" "${srcdir}/ref_tst_utf8.cdl"; ERR
echo "*** Accessing binary file ${UNISTRING}/tst_utf.nc..."
${NCDUMP} -h "${UNISTRING}/tst_utf.nc"; ERR
fi
echo "Test Passed. Cleaning up."
rm "${UNISTRING}/tst_utf.nc"; ERR
rmdir "${UNISTRING}"; ERR
rm -fr "${UNISTRING}"; ERR

53
ncdump/test_unicode_path.sh Executable file
View File

@ -0,0 +1,53 @@
#!/bin/sh
#
# Test to make sure ncdump works with a subdirectory which starts
# with a unicode character.
# See https://github.com/Unidata/netcdf-c/issues/1666 for more information.
# Ward Fisher
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
LC_ALL="C.UTF-8"
export LC_ALL
# Passing a utf8 name using either \x or actual characters
# to Visual Studio does not work well.
if test "x$FP_ISMSVC" = x ; then
#UNISTRING='\xe6\xb5\xb7'
UNISTRING='海'
else
UNISTRING='海'
fi
UNIFILE="tst_utf8_${UNISTRING}"
echo ""
echo "*** Generating netcdf-3 binary file ${UNIFILE}.nc ..."
${NCGEN} -3 -b -o "${UNIFILE}.nc" "${srcdir}/ref_tst_utf8.cdl"
echo "*** Accessing binary file ${UNIFILE}.nc ..."
${NCDUMP} -h "${UNIFILE}.nc"
if test "x$FEATURE_HDF5" = xyes ; then
echo "*** Generating netcdf-4 binary file ${UNIFILE}.nc ..."
rm -f "${UNIFILE}.nc"
${NCGEN} -4 -b -o "${UNIFILE}.nc" "${srcdir}/ref_tst_utf8.cdl"
echo "*** Accessing binary file ${UNIFILE}.nc ..."
${NCDUMP} -h "${UNIFILE}.nc"
fi
# This test was moved here from tst_nccopy4.sh
# to unify all the unicode path tests
echo "*** Test nccopy ${UNIFILe} copy_of_${UNIFILE} ..."
${NCCOPY} ${UNIFILE}.nc copy_of_${UNIFILE}.nc
${NCDUMP} -n copy_of_${UNIFILE} ${UNIFILE}.nc > tmp_${UNIFILE}.cdl
${NCDUMP} copy_of_${UNIFILE}.nc > copy_of_${UNIFILE}.cdl
echo "*** compare " with copy_of_${UNIFILE}.cdl
diff copy_of_${UNIFILE}.cdl tmp_${UNIFILE}.cdl
rm copy_of_${UNIFILE}.nc copy_of_${UNIFILE}.cdl tmp_${UNIFILE}.cdl
echo "Test Passed. Cleaning up."
rm ${UNIFILE}.nc

22
ncdump/tst_nccopy4.sh
View File

@ -7,7 +7,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
if test -f tst_comp2${ext} ; then ${execdir}/tst_comp2 ; fi
set -e
echo ""
@ -16,22 +15,27 @@ echo ""
# ref_tst_compounds2 ref_tst_compounds3 ref_tst_compounds4
TESTFILES='tst_comp tst_comp2 tst_enum_data tst_fillbug
tst_group_data tst_nans tst_opaque_data tst_solar_1 tst_solar_2
tst_solar_cmp tst_special_atts tst_string_data tst_unicode'
tst_solar_cmp tst_special_atts tst_string_data'
if test "x$NC_VLEN_NOTEST" = x ; then
TESFILES="$TESTFILES tst_vlen_data"
TESTFILES="$TESTFILES tst_vlen_data"
fi
echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files"
for i in $TESTFILES ; do
echo "*** Test nccopy $i.nc copy_of_$i.nc ..."
if test "x$i" = xtst_vlen_data ; then
ls -l tst_vlen_data*
ls -l *.nc
fi
${NCCOPY} $i.nc copy_of_$i.nc
${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl
${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl
# echo "*** compare " with copy_of_$i.cdl
diff copy_of_$i.cdl tmp.cdl
rm copy_of_$i.nc copy_of_$i.cdl tmp.cdl
${NCDUMP} -n copy_of_$i $i.nc > tmp_$i.cdl
${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl
echo "*** compare " with copy_of_$i.cdl
diff copy_of_$i.cdl tmp_$i.cdl
rm copy_of_$i.nc copy_of_$i.cdl tmp_$i.cdl
done
# echo "*** Testing compression of deflatable files ..."
./tst_compress
echo "*** Test nccopy -d1 can compress a classic format file ..."
@ -94,6 +98,8 @@ ${NCCOPY} -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc
${NCDUMP} -n tst_bug321 tmp.nc > tmp.cdl
diff -b $srcdir/tst_bug321.cdl tmp.cdl
rm tst_chunking.nc tmp.nc tmp.cdl tmp-chunked.nc tmp-chunked.cdl tmp-unchunked.nc tmp-unchunked.cdl
echo "*** Test that nccopy -c dim/n works as intended "
${NCGEN} -4 -b -o tst_perdimspecs.nc $srcdir/ref_tst_perdimspecs.cdl
${NCCOPY} -M0 -4 -c "time/10,lat/15,lon/20" tst_perdimspecs.nc tmppds.nc

9
ncdump/tst_netcdf4.sh
View File

@ -97,15 +97,6 @@ ${execdir}/tst_nans ; ERR
${NCDUMP} tst_nans.nc | sed 's/e+0/e+/g' > tst_nans.cdl ; ERR
diff -b tst_nans.cdl $srcdir/ref_tst_nans.cdl ; ERR
# Do unicode test only if it exists => BUILD_UTF8 is true
if test -f ./tst_unicode -o -f ./tst_unicode.exe ; then
echo "*** dumping tst_unicode.nc to tst_unicode.cdl..."
${execdir}/tst_unicode ; ERR
${NCDUMP} tst_unicode.nc | sed 's/e+0/e+/g' > tst_unicode.cdl ; ERR
#echo "*** comparing tst_unicode.cdl with ref_tst_unicode.cdl..."
#diff -b tst_unicode.cdl $srcdir/ref_tst_unicode.cdl
fi
echo "*** Running tst_special_atts.c to create test files."
${execdir}/tst_special_atts ; ERR
${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR

9
ncdump/tst_netcdf4_4.sh
View File

@ -72,13 +72,8 @@ ${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl
cleanncprops tst_ncf213.cdl tst_ncf213.tmp
cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp
# Now compare
ok=1;
if diff -b tst_ncf213.tmp ref_tst_ncf213.tmp ; then ok=1; else ok=0; fi
if test $ok = 0 ; then
echo "*** FAIL: NCF-213 Bug Fix test"
exit 1
fi
diff -b tst_ncf213.tmp ref_tst_ncf213.tmp
echo "*** All ncgen and ncdump extra test output for netCDF-4 format passed!"
exit 0

146
ncdump/tst_unicode.c
View File

@ -11,21 +11,33 @@
#include "err_macros.h"
#include <stdlib.h>
#include <stdio.h>
#include <netcdf.h>
#include "netcdf.h"
#include "ncpathmgr.h"
#ifdef _WIN32
#include <windows.h>
#include <direct.h>
#endif
#ifdef HAVE_LOCALE_H
#include <locale.h>
#endif
#define DEBUG
/* The data file we will create. */
#define FILE7_NAME "tst_unicode.nc"
static const unsigned char prefix[] = {
't','s','t','_','u','t','f','8','_',
'\xe6', '\xb5', '\xb7',
'\0'
};
/* Other meta-data */
#define UNITS "units"
#define NDIMS 1
#define UTF8_BYTES 18
int
main(int argc, char **argv)
{
int ncid, dimid, varid;
int dimids[NDIMS];
unsigned char name_utf8[] = {
static unsigned char name_utf8[] = {
0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */
0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */
0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */
@ -41,44 +53,12 @@ main(int argc, char **argv)
#define UNAME ((char *) name_utf8)
#define UNAMELEN (sizeof name_utf8)
char name_in[UNAMELEN + 1], strings_in[UNAMELEN + 1];
nc_type att_type;
size_t att_len;
printf("\n*** Testing UTF-8.\n");
printf("*** creating UTF-8 test file %s...", FILE7_NAME);
if (nc_create(FILE7_NAME, NC_NETCDF4, &ncid)) ERR;
/* Define dimension with Unicode UTF-8 encoded name */
if (nc_def_dim(ncid, UNAME, UTF8_BYTES, &dimid)) ERR;
dimids[0] = dimid;
/* Define variable with same name */
if (nc_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid)) ERR;
/* Create string attribute with same value */
if (nc_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME)) ERR;
if (nc_enddef(ncid)) ERR;
/* Write string data, UTF-8 encoded, to the file */
if (nc_put_var_text(ncid, varid, UNAME)) ERR;
if (nc_close(ncid)) ERR;
/* Check it out. */
/* Reopen the file. */
if (nc_open(FILE7_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_varid(ncid, UNAME, &varid)) ERR;
if (nc_inq_varname(ncid, varid, name_in)) ERR;
{
/* Note, name was normalized before storing, so retrieved name
won't match original unnormalized name. Check that we get
normalized version, instead. */
/* NFC normalized UTF-8 for Unicode 8-character "Hello" in Greek */
unsigned char norm_utf8[] = {
static unsigned char norm_utf8[] = {
0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */
0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */
0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */
@ -92,14 +72,84 @@ main(int argc, char **argv)
};
#define NNAME ((char *) norm_utf8)
#define NNAMELEN (sizeof norm_utf8)
if (strncmp(NNAME, name_in, NNAMELEN) != 0) ERR;
static int
check(int err, int line, const char* file)
{
if(err != 0) {
fprintf(stderr,"ERR %s.%d (%d) %s\n",file,line,err,nc_strerror(err));
fflush(stderr);
}
return err;
}
#define CHECK(err) {if((ret=check(err,__LINE__,__FILE__))) goto done;}
static int
test(int flags, const char* model)
{
int ret = NC_NOERR;
int ncid, dimid, varid;
int dimids[NDIMS];
char name_in[UNAMELEN + 1], strings_in[UNAMELEN + 1];
nc_type att_type;
size_t att_len;
char filename[4096];
/* Construct the file name */
snprintf(filename,sizeof(filename),"%s_%s.nc",prefix,model);
printf("\n*** Testing UTF-8: %s model\n",model);
printf("*** creating UTF-8 test file |%s|...", filename);
CHECK(nc_create(filename, flags, &ncid));
/* Define dimension with Unicode UTF-8 encoded name */
CHECK(nc_def_dim(ncid, UNAME, UTF8_BYTES, &dimid));
dimids[0] = dimid;
/* Define variable with same name */
CHECK(nc_def_var(ncid, UNAME, NC_CHAR, NDIMS, dimids, &varid));
/* Create string attribute with same value */
CHECK(nc_put_att_text(ncid, varid, UNITS, UNAMELEN, UNAME));
CHECK(nc_enddef(ncid));
/* Write string data, UTF-8 encoded, to the file */
CHECK(nc_put_var_text(ncid, varid, UNAME));
CHECK(nc_close(ncid));
/* Check it out. */
/* Reopen the file. */
CHECK(nc_open(filename, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, UNAME, &varid));
CHECK(nc_inq_varname(ncid, varid, name_in));
{
if (strncmp(NNAME, name_in, NNAMELEN) != 0)
{CHECK(NC_EBADNAME);}
}
if (nc_inq_att(ncid, varid, UNITS, &att_type, &att_len)) ERR;
if (att_type != NC_CHAR || att_len != UNAMELEN) ERR;
if (nc_get_att_text(ncid, varid, UNITS, strings_in)) ERR;
CHECK(nc_inq_att(ncid, varid, UNITS, &att_type, &att_len));
CHECK(att_type != NC_CHAR || att_len != UNAMELEN);
CHECK(nc_get_att_text(ncid, varid, UNITS, strings_in));
strings_in[att_len] = '\0'; /* null terminate, because nc_get_att_text doesn't */
if (strncmp(UNAME, strings_in, UNAMELEN) != 0) ERR;
if (nc_close(ncid)) ERR;
if (strncmp(UNAME, strings_in, UNAMELEN) != 0)
{CHECK(NC_EBADNAME);}
CHECK(nc_close(ncid));
done:
return ret;
}
int
main(int argc, char **argv)
{
/* Run the utf8 test both for netcdf-4 and netcdf-3 */
if(test(0,"classic")) ERR;
#ifdef USE_HDF5
if(test(NC_NETCDF4,"enhanced")) ERR;
#endif
SUMMARIZE_ERR;
FINAL_RESULTS;

8
ncgen/main.c
View File

@ -200,9 +200,7 @@ main(
int c;
FILE *fp;
struct Languages* langs;
#ifdef __hpux
setlocale(LC_CTYPE,"");
#endif
init_netcdf();
opterr = 1; /* print error message if bad option */
@ -353,7 +351,7 @@ main(
break;
case 'o': /* to explicitly specify output name */
if(netcdf_name) efree(netcdf_name);
netcdf_name = NCdeescape(optarg);
netcdf_name = NC_backslashUnescape(optarg);
break;
case 'P': /* diskless with persistence */
diskless = 1;
@ -462,7 +460,7 @@ main(
}
}
cdlname = nulldup(argv[0]);
cdlname = NC_backslashUnescape(argv[0]);
if(cdlname != NULL) {
if(strlen(cdlname) > NC_MAX_NAME)
cdlname[NC_MAX_NAME] = '\0';

1
ncgen/semantics.c
View File

@ -9,6 +9,7 @@
#include "dump.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
#include "ncpathmgr.h"
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))

8
ncgen3/Makefile.am
View File

@ -37,6 +37,7 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc
# This is used if someone wants to rebuild the parser and lexer
# Otherwise never invoked, but records how to do it.
makeparser::
rm -f ncgenl.c lex.ncg.c
flex -Pncg -8 ncgen.l
rm -f ncgenl.c
sed -e s/lex.ncg.c/ncgenl.c/g <lex.ncg.c >ncgenl.c
@ -45,10 +46,3 @@ makeparser::
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.c >ncgeny.c
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.h >ncgeny.h
rm -f ncgen.tab.c ncgen.tab.h
makeparserold::
flex -Pncg -8 ncgen.l
mv lex.ncg.c ncgenyy.c
bison -pncg -d ncgen.y
mv ncgen.tab.c ncgentab.c
mv ncgen.tab.h ncgentab.h

11
ncgen3/main.c
View File

@ -20,10 +20,6 @@
#define snprintf _snprintf
#endif
#ifdef __hpux
#include <locale.h>
#endif
#include "netcdf.h"
#include "ncpathmgr.h"
@ -91,10 +87,6 @@ main(
int c;
FILE *fp;
#ifdef __hpux
setlocale(LC_CTYPE,"");
#endif
#ifdef MDEBUG
malloc_debug(2) ; /* helps find malloc/free errors on Sun */
#endif /* MDEBUG */
@ -213,6 +205,9 @@ main(
argv += optind;
if (argc > 1) {
int i;
for(i=0;i<argc;i++)
fprintf(stderr,"xarg(%d): |%s|\n",i,argv[i]);
derror ("%s: only one input file argument permitted",progname);
return(6);
}

2
ncgen3/ncgen.y
View File

@ -898,5 +898,3 @@ clearout(void) /* reset symbol table to empty */
#define YY_NO_INPUT
#include "ncgenl.c"

2
ncgen3/ncgeny.c
View File

@ -2487,5 +2487,3 @@ clearout(void) /* reset symbol table to empty */
#define YY_NO_INPUT
#include "ncgenl.c"

724
nczarr_test/itmeta.c
View File

@ -1,724 +0,0 @@
/*
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "ut_includes.h"
#define SETUP
#define DEBUG
#define FILE "testmeta.ncz"
#define VAR1 "var1"
#define VAR2 "var2"
static char* url = NULL;
/* Arguments from command line */
struct Options {
int debug;
char* cmd;
} options;
typedef union AtomicsR {
char bytev[1];
short shortv[1];
int intv[1];
float floatv[1];
double doublev[1];
unsigned char ubytev[1];
unsigned short ushortv[1];
unsigned int uintv[1];
long long int64v[1];
unsigned long long uint64v[1];
char charv[1];
} AtomicsR;
typedef struct AtomicsW {
char bytev[1];
short shortv[1];
int intv[1];
float floatv[1];
double doublev[1];
unsigned char ubytev[1];
unsigned short ushortv[1];
unsigned int uintv[1];
long long int64v[1];
unsigned long long uint64v[1];
char charv[1];
} AtomicsW;
/* Forward */
static int testcreate(void);
static int testdim1(void);
static int testvar1(void);
static int testvar2(void);
static int testattr1(void);
static int testattr2(void);
static int testread1(void);
static int testread2(void);
static int testread3(void);
struct Test {
const char* cmd;
int (*test)(void);
} tests[] = {
{"create", testcreate},
{"dim1", testdim1},
{"var1", testvar1},
{"var2", testvar2},
{"attr1", testattr1},
{"attr2", testattr2},
{"read1", testread1},
{"read2", testread2},
{"read3", testread3},
{NULL, NULL}
};
#ifdef SETUP
#define NCCHECK(expr) nccheck((expr),__LINE__)
void nccheck(int stat, int line)
{
if(stat) {
fprintf(stderr,"%d: %s\n",line,nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
#endif
static void
makeurl(const char* file)
{
char wd[4096];
NCbytes* buf = ncbytesnew();
ncbytescat(buf,"file://");
(void)getcwd(wd, sizeof(wd));
ncbytescat(buf,wd);
ncbytescat(buf,"/");
ncbytescat(buf,file);
ncbytescat(buf,"#mode=zarr"); /* => use default file: format */
url = ncbytesextract(buf);
ncbytesfree(buf);
}
int
main(int argc, char** argv)
{
int stat = NC_NOERR;
int c;
struct Test* t = NULL;
struct Test* test = NULL;
char* file = NULL;
memset((void*)&options,0,sizeof(options));
while ((c = getopt(argc, argv, "dc:o:")) != EOF) {
switch(c) {
case 'd':
options.debug = 1;
break;
case 'c':
if(options.cmd != NULL) {
fprintf(stderr,"error: multiple tests specified\n");
stat = NC_EINVAL;
goto done;
}
options.cmd = strdup(optarg);
break;
case 'o':
if(file != NULL) {
fprintf(stderr,"error: multiple output files specified\n");
stat = NC_EINVAL;
goto done;
}
file = strdup(optarg);
break;
case '?':
fprintf(stderr,"unknown option\n");
stat = NC_EINVAL;
goto done;
}
}
if(options.cmd == NULL) {
fprintf(stderr,"no command specified\n");
stat = NC_EINVAL;
goto done;
}
if(file == NULL)
file = strdup(FILE);
makeurl(file);
#ifdef DEBUG
fprintf(stderr,"test: %s url=|%s|\n",options.cmd,url);
fflush(stderr);
#endif
/* Find the corresponding test for command */
for(t=tests;t->cmd;t++) {
if(strcmp(options.cmd,t->cmd)==0) {test = t; break;}
}
if(test == NULL) {
fprintf(stderr,"unknown command: %s\n",optarg);
stat = NC_EINVAL;
goto done;
}
/* Execute */
test->test();
done:
if(stat)
nc_strerror(stat);
return (stat ? 1 : 0);
}
/* Create test netcdf4 file via netcdf.h API*/
static int
testcreate(void)
{
int stat = NC_NOERR;
int ncid;
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Create file and add a dimension */
static int
testdim1(void)
{
int stat = NC_NOERR;
int ncid, dimid;
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
if((stat = nc_def_dim(ncid, "dim1", (size_t)1, &dimid)))
goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Create file and add a variable */
static int
testvar1(void)
{
int stat = NC_NOERR;
int ncid, varid;
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
if((stat = nc_def_var(ncid, VAR1, NC_INT, 0, NULL, &varid)))
goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Create file and add a rank 2 variable */
static int
testvar2(void)
{
int stat = NC_NOERR;
int ncid, varid1, varid2;
int dimids[2];
int ndims;
int fillval;
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
ndims = 0;
if((stat = nc_def_dim(ncid, "dim2", 2, &dimids[0])))
goto done;
ndims++;
if((stat = nc_def_dim(ncid, "dim3", 3, &dimids[1])))
goto done;
ndims++;
if((stat = nc_def_var(ncid, VAR1, NC_INT, ndims, dimids, &varid1)))
goto done;
/* make VAR1 no-fill */
if((stat = nc_def_var_fill(ncid,varid1,NC_NOFILL,NULL)))
goto done;
/* make VAR1 contiguous */
if((stat = nc_def_var_chunking(ncid,varid1,NC_CONTIGUOUS,NULL)))
goto done;
/* Define var2 as chunked, but use default chunking */
if((stat = nc_def_var(ncid, VAR2, NC_INT, ndims, dimids, &varid2)))
goto done;
if((stat = nc_def_var_chunking(ncid,varid2,NC_CHUNKED,NULL)))
goto done;
/* make VAR1 use non-default fill */
fillval = 137;
if((stat = nc_def_var_fill(ncid,varid2,NC_FILL,&fillval)))
goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Create file and add global attributes of all supported types */
static int
testattr1(void)
{
int stat = NC_NOERR;
int ncid;
AtomicsW atoms = {
{-17},
{-17},
{-17},
{17.0f},
{17.0},
{17u},
{17u},
{17u},
{17ll},
{17ull},
{'7'}};
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"byte_a",NC_BYTE,1,atoms.bytev))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"short_a",NC_SHORT,1,atoms.shortv))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"int_a",NC_INT,1,atoms.intv))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"float_a",NC_FLOAT,1,atoms.floatv))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"double_a",NC_DOUBLE,1,atoms.doublev))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"ubyte_a",NC_UBYTE,1,atoms.ubytev))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"ushort_a",NC_USHORT,1,atoms.ushortv))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"uint_a",NC_UINT,1,atoms.uintv))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"int64_a",NC_INT64,1,atoms.int64v))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"uint64_a",NC_UINT64,1,atoms.uint64v))) goto done;
if((stat = nc_put_att(ncid,NC_GLOBAL,"char_a",NC_CHAR,1,atoms.charv))) goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Create file and a var and add some attributes to the var */
static int
testattr2(void)
{
int stat = NC_NOERR;
int ncid, varid1;
int intv[1] = {-17};
float floatv[1] = {17.0f};
long long int64v[1] = {17ll};
char charv[1] = {'7'};
unlink(FILE);
if((stat = nc_create(url, 0, &ncid)))
goto done;
if((stat = nc_def_var(ncid, VAR1, NC_INT, 0, NULL, &varid1)))
goto done;
if((stat = nc_put_att(ncid,varid1,"int_a",NC_INT,1,intv))) goto done;
if((stat = nc_put_att(ncid,varid1,"float_a",NC_FLOAT,1,floatv))) goto done;
if((stat = nc_put_att(ncid,varid1,"int64_a",NC_INT64,1,int64v))) goto done;
if((stat = nc_put_att(ncid,varid1,"char_a",NC_CHAR,1,charv))) goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
static int
fv2string(nc_type xtype, size_t xtypesize, void* fill_value, char** fillstringp)
{
char sval[1024];
switch (xtype) {
case NC_BYTE:
snprintf(sval,sizeof(sval),"%d",*((char*)fill_value));
break;
case NC_CHAR:
snprintf(sval,sizeof(sval),"'%c'",*((char*)fill_value));
break;
case NC_SHORT:
snprintf(sval,sizeof(sval),"%d",*((short*)fill_value));
break;
case NC_INT:
snprintf(sval,sizeof(sval),"%d",*((int*)fill_value));
break;
case NC_FLOAT:
snprintf(sval,sizeof(sval),"%g",*((float*)fill_value));
break;
case NC_DOUBLE:
snprintf(sval,sizeof(sval),"%lg",*((double*)fill_value));
break;
case NC_UBYTE:
snprintf(sval,sizeof(sval),"%u",*((unsigned char*)fill_value));
break;
case NC_USHORT:
snprintf(sval,sizeof(sval),"%u",*((unsigned short*)fill_value));
break;
case NC_UINT:
snprintf(sval,sizeof(sval),"%u",*((unsigned int*)fill_value));
break;
case NC_INT64:
snprintf(sval,sizeof(sval),"%lld",*((long long*)fill_value));
break;
case NC_UINT64:
snprintf(sval,sizeof(sval),"%llu",*((unsigned long long*)fill_value));
break;
default:
strlcat(sval,"<unknown>",sizeof(sval));
break;
}
if(fillstringp) *fillstringp = strdup(sval);
return NC_NOERR;
}
static int
reportvar(int ncid, int varid)
{
int i,stat = NC_NOERR;
char varname[NC_MAX_NAME];
nc_type xtype;
int ndims, natts;
char typename[NC_MAX_NAME];
size_t typesize;
int dimids[NC_MAX_VAR_DIMS];
char* dimname[NC_MAX_VAR_DIMS];
size_t dimsize[NC_MAX_VAR_DIMS];
int no_fill;
void* fill_value = NULL;
int shuffle, deflate, deflate_level;
int fletcher32;
int endian;
unsigned int* filterids
size_t nfilters;
int storage;
size_t chunksizes[NC_MAX_VAR_DIMS];
const char* tmp = NULL;
char* fillstring = NULL;
if((stat = nc_inq_var(ncid,varid,varname,&xtype,&ndims,dimids,&natts)))
goto done;
/* get basetype name and size*/
if((stat = nc_inq_type(ncid, xtype, typename, &typesize))) goto done;
/* Get dimension sizes */
for(i=0;i<ndims;i++) {
char name[NC_MAX_NAME];
if((stat = nc_inq_dim(ncid, dimids[i], name, &dimsize[i])))
goto done;
dimname[i] = strdup(name);
}
if((stat = nc_inq_var_endian(ncid, varid, &endian)))
goto done;
if((stat = nc_inq_var_deflate(ncid, varid, &shuffle, &deflate, &deflate_level)))
goto done;
if((stat = nc_inq_var_fletcher32(ncid, varid, &fletcher32)))
goto done;
if((stat = nc_inq_var_filters(ncid, varid, &nfilters, NULL)))
goto done;
filterids = NULL;
if(nfilters > 0)
filterids = malloc(sizeof(unsigned int)*nfilters);
if((stat = nc_inq_var_filters(ncid, varid, &nfilters, filterids)))
goto done;
/* Allocate space for fill_value */
if((fill_value = calloc(1,typesize)) == NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = nc_inq_var_fill(ncid, varid, &no_fill, fill_value)))
goto done;
if((stat = nc_inq_var_chunking(ncid, varid, &storage, chunksizes)))
goto done;
/* Now dump everything */
printf("%s %s",typename, varname);
printf("(");
for(i=0;i<ndims;i++) {
if(i > 0) printf(",");
printf("%s=%lu",dimname[i],(unsigned long)dimsize[i]);
}
printf(");\n");
switch (endian) {
case NC_ENDIAN_NATIVE: tmp = "native"; break;
case NC_ENDIAN_LITTLE: tmp = "little"; break;
case NC_ENDIAN_BIG: tmp = "big"; break;
default: tmp = "unknown"; break;
}
printf("\t_Endianness=%s\n",tmp);
printf("\t_Fletcher32=%s\n",(fletcher32?"true":"false"));
printf("\t_Shuffle=%s\n",(shuffle?"true":"false"));
if(deflate) printf("\t_DeflateLevel=%d\n",deflate_level);
if(filterids != NULL) {
printf("\t_Filters=(");
for(i=0;i<nfilters;i++) {
if(i>0) printf(",");
printf("%u",filterids[i]);
}
printf(")\n");
}
if((stat=fv2string(xtype,typesize,fill_value,&fillstring))) goto done;
printf("\t_FillValue=%s\n",fillstring);
if(storage == NC_CONTIGUOUS) {
printf("\t_Storage=contiguous\n");
} else { /* chunked */
printf("\t_Storage=chunked\n");
printf("\t_ChunkSizes=");
for(i=0;i<ndims;i++) {
if(i>0) printf(",");
printf("%lu",(unsigned long)chunksizes[i]);
}
printf("\n");
}
done:
return stat;
}
/* Report variables from testvar2 */
static int
testread1(void)
{
int stat = NC_NOERR;
int ncid, varid1, varid2;
int ndims,nvars,natts,unlimdimid;
unlink(FILE);
#ifdef SETUP
NCCHECK(testvar2());
#endif
if((stat = nc_open(url, 0, &ncid)))
goto done;
/* Report file level info */
if((stat = nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)))
goto done;
printf("file info: ndims=%d nvars=%d natts=%d unlimdimid=%d\n",
ndims,nvars,natts,unlimdimid);
if((stat = nc_inq_varid(ncid, VAR1, &varid1))) goto done;
/* Report info about the var */
if((stat = reportvar(ncid,varid1))) goto done;
if((stat = nc_inq_varid(ncid, VAR2, &varid2))) goto done;
/* Report info about the var */
if((stat = reportvar(ncid,varid2))) goto done;
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
static int
reportatt(int ncid, int varid, int natts, int index)
{
int i,stat = NC_NOERR;
void* data = NULL;
char aname[NC_MAX_NAME];
nc_type atype;
size_t alen;
char typename[NC_MAX_NAME];
size_t typesize;
char* p = NULL; /* char* so we can do arithmetic */
/* Get some meta info about the attribute */
if((stat = nc_inq_attname(ncid,varid,index,aname)))
goto done;
if((stat = nc_inq_att(ncid, varid, aname, &atype, &alen))) goto done;
/* Get type info about atype */
if((stat = nc_inq_type(ncid, atype, typename, &typesize))) goto done;
/* Allocate space to read the attribute */
if((data = malloc(typesize * alen)) == NULL)
{stat = NC_ENOMEM; goto done;}
/* Read the attribute */
if((stat = nc_get_att(ncid,varid,aname,data))) goto done;
/* Now report */
printf("\t[%d] %s:%s(%lu) = ",index,typename,aname,(unsigned long)alen);
for(p=(char*)data,i=0;i<alen;i++,p+=typesize) {
AtomicsR* atom = (AtomicsR*)p;
printf("%s",(i==0?"{":" "));
switch (atype) {
case NC_BYTE: printf("%db",atom->bytev[0]); break;
case NC_SHORT: printf("%ds",atom->shortv[0]); break;
case NC_INT: printf("%d",atom->intv[0]); break;
case NC_FLOAT: printf("%gf",atom->floatv[0]); break;
case NC_DOUBLE: printf("%lg",atom->doublev[0]); break;
case NC_UBYTE: printf("%uub",atom->ubytev[0]); break;
case NC_USHORT: printf("%uus",atom->ushortv[0]); break;
case NC_UINT: printf("%uu",atom->intv[0]); break;
case NC_INT64: printf("%lldll",atom->int64v[0]); break;
case NC_UINT64: printf("%lluull",atom->uint64v[0]); break;
case NC_CHAR: printf("'%c'",atom->charv[0]); break;
}
}
printf("}\n");
done:
return stat;
}
/* Print out selected reserved attributes by name */
static int
reportreserved(int ncid)
{
int stat = NC_NOERR;
char* ncpropdata = NULL;
size_t len;
int superblockversion;
int isnetcdf4;
nc_type xtype;
/* Get and print _NCProperties */
if((stat = nc_inq_attlen(ncid, NC_GLOBAL, "_NCProperties", &len)))
goto done;
if((ncpropdata = calloc(1,len+1)) == NULL) {stat = NC_ENOMEM; goto done;}
if((stat = nc_get_att(ncid,NC_GLOBAL, "_NCProperties", ncpropdata)))
goto done;
ncpropdata[len] = '\0'; /* nul term */
printf("\t%s:%s(%lu) = |%s|\n", "char","_NCProperties", (unsigned long)len, ncpropdata);
/* Get and print _SuperblockVersion */
if((stat = nc_inq_att(ncid, NC_GLOBAL, "_SuperblockVersion", &xtype,&len)))
goto done;
if(len != 1 || xtype != NC_INT) {stat = NC_ENCZARR; goto done;}
if((stat = nc_get_att_int(ncid,NC_GLOBAL, "_SuperblockVersion", &superblockversion)))
goto done;
printf("\t%s:%s(%lu) = %d\n", "int","_SuperblockVersion", (unsigned long)len, superblockversion);
/* Get and print _Isnetcdf4 */
if((stat = nc_inq_att(ncid, NC_GLOBAL, "_IsNetcdf4", &xtype,&len)))
goto done;
if(len != 1 || xtype != NC_INT) {stat = NC_ENCZARR; goto done;}
if((stat = nc_get_att_int(ncid,NC_GLOBAL, "_IsNetcdf4", &isnetcdf4)))
goto done;
printf("\t%s:%s(%lu) = %d\n", "int","_IsNetcdf4", (unsigned long)len, isnetcdf4);
done:
nullfree(ncpropdata);
return stat;
}
/* Read the global attributes in testattr1 */
static int
testread2(void)
{
int i,stat = NC_NOERR;
int ncid;
int natts;
unlink(FILE);
#ifdef SETUP
NCCHECK(testattr1());
#endif
if((stat = nc_open(url, 0, &ncid)))
goto done;
printf("Reserved Attributes:\n");
if((stat = reportreserved(ncid))) goto done;
fflush(stdout);
printf("\nGlobal Attributes:\n");
/* Get and print the global attributes */
if((stat = nc_inq(ncid, NULL, NULL, &natts, NULL))) goto done;
for(i=0;i<natts;i++) {
if((stat = reportatt(ncid,NC_GLOBAL,natts,i))) goto done;
}
fflush(stdout);
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}
/* Read the variable attributes in testattr2 */
static int
testread3(void)
{
int i,stat = NC_NOERR;
int ncid, varid1;
int natts;
unlink(FILE);
#ifdef SETUP
NCCHECK(testattr2());
#endif
if((stat = nc_open(url, 0, &ncid)))
goto done;
/* Get and print the variable attributes */
printf("\nVariable Attributes:\n");
if((stat = nc_inq_varid(ncid, VAR1, &varid1))) goto done;
if((stat = nc_inq_var(ncid,varid1,NULL,NULL,NULL,NULL,&natts)))
goto done;
for(i=0;i<natts;i++) {
if((stat = reportatt(ncid,varid1,natts,i))) goto done;
}
fflush(stdout);
if((stat = nc_close(ncid)))
goto done;
done:
return THROW(stat);
}

2
nczarr_test/ncdumpchunks.c
View File

@ -485,7 +485,7 @@ main(int argc, char** argv)
}
{
char* s = NCdeescape(argv[0]);
char* s = NC_backslashUnescape(argv[0]);
strcpy(format.file_name,filenamefor(s));
nullfree(s);
}

2
nczarr_test/s3util.c
View File

@ -163,7 +163,7 @@ main(int argc, char** argv)
dumpoptions.nctype = typefor(optarg);
break;
case 'u': {
char* p = NCdeescape(optarg);
char* p = NC_backslashUnescape(optarg);
ncuriparse(p,&dumpoptions.url);
nullfree(p);
if(dumpoptions.url == NULL) {

2
nczarr_test/tst_utils.c
View File

@ -129,7 +129,7 @@ getoptions(int* argcp, char*** argvp)
*argvp += optind;
if(*argcp > 0) {
char* p = NCdeescape((*argvp)[0]);
char* p = NC_backslashUnescape((*argvp)[0]);
strcpy(options->file,filenamefor(p));
nullfree(p);
}

26
nczarr_test/ut_test.c
View File

@ -111,7 +111,7 @@ static void
getpathcwd(char** cwdp)
{
char buf[4096];
(void)NCcwd(buf,sizeof(buf));
(void)NCgetcwd(buf,sizeof(buf));
if(cwdp) *cwdp = strdup(buf);
}
@ -132,10 +132,14 @@ canonicalfile(char** fp)
f = *fp;
len = strlen(f);
if(len <= 1) return;
if(f[0] == '/' || f[0] == '\\' || hasdriveletter(f))
return; /* its already absolute */
ncuriparse(f,&uri);
if(uri != NULL) {ncurifree(uri); return;} /* its a url */
#if 1
abspath = NCpathabsolute(f);
#else
if(f[0] == '/' || f[0] == '\\' || hasdriveletter(f))
return; /* its already absolute */
#ifdef _WIN32
for(p=f;*p;p++) {if(*p == '\\') {*p = '/';}}
#endif
@ -166,9 +170,11 @@ canonicalfile(char** fp)
if(fwin32)
for(p=abspath;*p;p++) {if(*p == '/') {*p = '\\';}}
#endif
nullfree(f);
*fp = abspath;
nullfree(cwd);
#endif
nullfree(f);
fprintf(stderr,"canonicalfile: %s\n",abspath);
*fp = abspath;
}
void
@ -188,16 +194,19 @@ makeurl(const char* file, NCZM_IMPL impl)
NCbytes* buf = ncbytesnew();
NCURI* uri = NULL;
const char* kind = impl2kind(impl);
char* path = NULL;
char* urlpath = NULL;
char* p;
if(file && strlen(file) > 0) {
switch (impl) {
case NCZM_FILE:
case NCZM_ZIP:
/* Massage file to make it usable as URL path */
if((path = NCurlpath(file))==NULL) return NULL;
urlpath = strdup(file);
for(p=urlpath;*p;p++) {if(*p == '\\') *p = '/';}
ncbytescat(buf,"file://");
ncbytescat(buf,path);
ncbytescat(buf,urlpath);
nullfree(urlpath); urlpath = NULL;
ncbytescat(buf,"#mode=nczarr"); /* => use default file: format */
ncbytescat(buf,",");
ncbytescat(buf,kind);
@ -217,7 +226,6 @@ makeurl(const char* file, NCZM_IMPL impl)
}
ncurifree(uri);
ncbytesfree(buf);
nullfree(path);
fprintf(stderr,"url=|%s|\n",url);
fflush(stderr);
return url;

2
nczarr_test/zmapio.c
View File

@ -177,7 +177,7 @@ main(int argc, char** argv)
}
{
char* p = NCdeescape(argv[0]);
char* p = NC_backslashUnescape(argv[0]);
strcpy(dumpoptions.infile,filenamefor(p));
if(p) free(p);
}

6
nczarr_test/zs3parse.c
View File

@ -87,7 +87,7 @@ main(int argc, char** argv)
fprintf(stderr, "zs3parse: no url|file specified\n");
goto fail;
}
s3options.url = NCdeescape(argv[0]);
s3options.url = strdup(argv[0]);
stat = processurl(s3options.op, s3options.url, &piece);
if(stat == NC_NOERR) {
@ -126,10 +126,6 @@ processurl(S3op op, const char* surl, char** piece)
&& strcmp(url->protocol,"http") != 0)
{stat = NC_EURL; goto done;}
/* Path better look absolute */
if(!nczm_isabsolutepath(url->path))
{stat = NC_EURL; goto done;}
if(url->host == NULL || strlen(url->host) == 0)
{stat = NC_EURL; goto done;}
if((host = strdup(url->host))==NULL)

5
test_common.in
View File

@ -8,6 +8,11 @@
# Define location of execution
TOPSRCDIR='@abs_top_srcdir@'
TOPBUILDDIR='@abs_top_builddir@'
FP_ISCMAKE=@ISCMAKE@
FP_ISMSVC=@ISMSVC@
# Feature flags
FEATURE_HDF5=@HAS_HDF5@
# Define selected features of the build
FEATURE_HDF5=@HAS_HDF5@

126
unit_test/test_pathcvt.c
View File

@ -10,61 +10,113 @@ Test the NCpathcvt
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "netcdf.h"
#include "ncpathmgr.h"
#undef VERBOSE
#define DEBUG
#define NKINDS 4
static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};
typedef struct Test {
char* path;
char* expected;
char* test;
char* expected[NKINDS];
} Test;
/* Path conversion tests */
static Test PATHTESTS[] = {
#ifdef _MSC_VER
{"/xxx/a/b","/xxx/a/b"},
{"d:/x/y","d:\\x\\y"},
{"/cygdrive/d/x/y","d:\\x\\y"},
{"/d/x/y","d:\\x\\y"},
{"/cygdrive/d","d:\\"},
{"/d","d:\\"},
{"/cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","d:\\git\\netcdf-c\\dap4_test\\daptestfiles\\test_anon_dim.2.syn"},
{"[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"},
#else
{"/xxx/a/b","/xxx/a/b"},
{"d:/x/y","d:/x/y"},
{"/cygdrive/d/x/y","d:/x/y"},
{"/d/x/y","d:/x/y"},
{"/cygdrive/d","d:/"},
{"/d","d:/"},
{"/cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","d:/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"},
{"[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn","[dap4]file:///cygdrive/d/git/netcdf-c/dap4_test/daptestfiles/test_anon_dim.2.syn"},
{"/xxx/a/b",{"/xxx/a/b", "/c/xxx/a/b", "/cygdrive/c/xxx/a/b", "c:\\xxx\\a\\b"}},
{"d:/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}},
{"/cygdrive/d/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}},
{"/d/x/y",{ "/d/x/y", "/d/x/y", "/cygdrive/d/x/y", "d:\\x\\y"}},
{"/cygdrive/d",{ "/d", "/d", "/cygdrive/d", "d:"}},
{"/d", {"/d", "/d", "/cygdrive/d", "d:"}},
{"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",{
"/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",
"/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",
"/cygdrive/d/git/netcdf-c/dap4_test/test_anon_dim.2.syn",
"d:\\git\\netcdf-c\\dap4_test\\test_anon_dim.2.syn"}},
/* Test relative path */
{"x/y",{ "x/y", "x/y", "x/y", "x\\y"}},
#ifndef _WIN32
/* Test utf8 path */
{"/海/海",{ "/海/海", "/c/海/海", "/cygdrive/c/海/海", "c:\\海\\海"}},
#endif
{NULL,NULL}
{NULL, {NULL, NULL, NULL, NULL}}
};
/*Forward */
static const char* kind2string(int kind);
int
main(int argc, char** argv)
{
Test* test;
int failcount = 0;
char* cvt = NULL;
int k;
int drive = 'c';
for(test=PATHTESTS;test->path;test++) {
char* cvt = NCpathcvt(test->path);
if(cvt == NULL) {
fprintf(stderr,"TEST returned NULL: %s\n",test->path);
exit(1);
}
if(strcmp(cvt,test->expected) != 0) {
fprintf(stderr,"NCpathcvt failed:: input: |%s| expected=|%s| actual=|%s|\n",test->path,test->expected,cvt);
failcount++;
}
#ifdef VERBOSE
fprintf(stderr,"NCpathcvt:: input: |%s| actual=|%s|\n",test->path,cvt);
nc_initialize();
/* Test localkind X path kind */
for(k=0;k<NKINDS;k++) {
int kind = kinds[k];
/* Iterate over the test paths */
for(test=PATHTESTS;test->test;test++) {
/* Compare output for the localkind */
if(test->expected[k] == NULL) {
#ifdef DEBUG
fprintf(stderr,"TEST local=%s: %s ignored\n",kind2string(kind),test->test);
#endif
free(cvt);
continue;
}
cvt = NCpathcvt_test(test->test,kind,drive);
#ifdef DEBUG
fprintf(stderr,"TEST local=%s: input: |%s| expected=|%s| actual=|%s|: ",
kind2string(kind),test->test,test->expected[k],cvt);
#endif
fflush(stderr); fflush(stdout);
if(cvt == NULL) {
#ifdef DEBUG
fprintf(stderr," ILLEGAL");
#endif
failcount++;
} else if(strcmp(cvt,test->expected[k]) != 0) {
#ifdef DEBUG
fprintf(stderr," FAIL");
#endif
failcount++;
} else {
#ifdef DEBUG
fprintf(stderr," PASS");
#endif
}
#ifdef DEBUG
fprintf(stderr,"\n");
#endif
nullfree( cvt); cvt = NULL;
}
}
fprintf(stderr,"%s test_ncuri\n",failcount > 0 ? "***FAIL":"***PASS");
nullfree(cvt);
fprintf(stderr,"%s test_pathcvt\n",failcount > 0 ? "***FAIL":"***PASS");
nc_finalize();
return (failcount > 0 ? 1 : 0);
}
static const char*
kind2string(int kind)
{
switch(kind) {
case NCPD_NIX:
return "Linux";
case NCPD_MSYS:
return "MSYS";
case NCPD_CYGWIN:
return "Cygwin";
case NCPD_WIN:
return "Windows";
default: break;
}
return "unknown";
}