From 9f44f64d05b994b1a67ebd4cbdeb8c087ad420ea Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 13 May 2011 20:58:40 +0000 Subject: [PATCH] See Jira issues NCF-48 and CDM-57 --- man4/netcdf.texi | 188 +++++++++++++++++++++++++++++++++++++++------ oc/curlfunctions.c | 6 ++ oc/ocinternal.c | 44 ++++++----- oc/ocinternal.h | 1 + oc/rc.c | 67 +++++++++++----- oc/rc.h | 2 +- 6 files changed, 243 insertions(+), 65 deletions(-) diff --git a/man4/netcdf.texi b/man4/netcdf.texi index a93902d03..801a5ca4a 100644 --- a/man4/netcdf.texi +++ b/man4/netcdf.texi @@ -3910,35 +3910,177 @@ Also, if you are accessing data over an NFS mount, you may see some .nfsxxxxx files; those can be ignored as well. -@subsection ESG SSL Support -Limited support for SSL is provided via parameters in the -``.dodsrc'' configuration file as provided by the oc library -(see @uref{http://opendap.org/download/oc.html}), which -is included as part of the standard netCDF distribution. -Note that the SSL support was added at the -request of the Earth System Grid (ESG) and as such -it provides the minimum needed for accessing ESG data. +@subsection HTTP Configuration. -The .dodsrc parameters needed to access SSL -are the following: +Limited support for configuring the http connection +is provided via parameters in the +``.httprc'' configuration file. Although deprecated, the name +``.dodsrc'' may also be used. +The relevant .httprc file is located by first looking in the +current working directory, and if not found, then looking in the +directory specified by the ``$HOME'' environment variable. +Entries in the .httprc file are of the form: +@example +['['']']= +@end example +That is, it consists of a key name and value pair +and optionally preceded by a url enclosed in square +brackets. + +For given KEY and URL strings, the value chosen is as follows: +@enumerate +@item If URL is null, then look for the .dodsrc entry that has no url prefix and whose key is same as the KEY for which we are looking. + +@item If the URL is not null, then look for all the .dodsrc entries +that have a url, URL1, say, and for which URL1 is a prefix (in the +string sense) of URL. For example, if URL = http//x.y/a, then it will +match entries of the form +@example +1. [http//x.y/a]KEY=VALUE +2. [http//x.y/a/b]KEY=VALUE +@end example +It will not match an entry of the form +@example +[http//x.y/b]KEY=VALUE +@end example +because ``http://x.y/b'' is not a string prefix of ``http://x.y/a''. +Finally from the set so constructed, choose the entry with the longest +url prefix: ``http//x.y/a/b]KEY=VALUE'' in this case. +@end enumerate + +Currently, the supported set of keys (with descriptions) are as follows. @itemize -@item CURL.SSL.VALIDATE -@item CURL.COOKIEJAR -@item CURL.SSL.CERTIFICATE -@item CURL.SSL.KEY -@item CURL.SSL.CAPATH +@item HTTP.VERBOSE + @enumerate + @item Type: boolean ("1"/"0") + @item Description: +Produce verbose output, especially using SSL. + @item Related CURL Flags: CURLOPT_VERBOSE + @end enumerate + +@item HTTP.DEFLATE + @enumerate + @item Type: boolean ("1"/"0") + @item Description: +Allow use of compression by the server. + @item Related CURL Flags: CURLOPT_ENCODING + @end enumerate + +@item HTTP.COOKIEJAR + @enumerate + @item Type: String representing file path + @item Description: +Specify the name of file into which to store cookies. +Defaults to in-memory storage. + @item Related CURL Flags:CURLOPT_COOKIEJAR + @end enumerate + +@item HTTP.COOKIEFILE + @enumerate + @item Type: String representing file path + @item Description: +Same as HTTP.COOKIEJAR. + @item Related CURL Flags: CURLOPT_COOKIEFILE + @end enumerate + +@item HTTP.CREDENTIALS.USER + @enumerate + @item Type: String representing user name + @item Description: +Specify the user name for Digest and Basic authentication. + @item Related CURL Flags: + @end enumerate + +@item HTTP.CREDENTIALS.PASSWORD + @enumerate + @item Type: String representing password + @item Type: boolean ("1"/"0") + @item Description: +Specify the password for Digest and Basic authentication. + @item Related CURL Flags: + @end enumerate + +@item HTTP.SSL.CERTIFICATE + @enumerate + @item Type: String representing file path + @item Description: +Path to a file containing a PEM cerficate. + @item Related CURL Flags: CURLOPT_CERT + @end enumerate + +@item HTTP.SSL.KEY + @enumerate + @item Type: String representing file path + @item Description: +Same as HTTP.SSL.CERTIFICATE, and should usually have the same value. + @item Related CURL Flags: CURLOPT_SSLKEY + @end enumerate + +@item HTTP.SSL.KEYPASSWORD + @enumerate + @item Type: String representing password + @item Description: +Password for accessing the HTTP.SSL.KEY/HTTP.SSL.CERTIFICATE + @item Related CURL Flags: CURLOPT_KEYPASSWORD + @end enumerate + +@item HTTP.SSL.CAPATH + @enumerate + @item Type: String representing directory + @item Description: +Path to a directory containing trusted certificates for validating +server sertificates. + @item Related CURL Flags: CURLOPT_CAPATH + @end enumerate + +@item HTTP.SSL.VALIDATE + @enumerate + @item Type: boolean ("1"/"0") + @item Description: +Cause the client to verify the server's presented certificate. + @item Related CURL Flags: CURLOPT_SSL_VERIFYPEER, CURLOPT_SSL_VERIFYHOST + @end enumerate + +@item HTTP.TIMEOUT + @enumerate + @item Type: String ("dddddd") + @item Description: +Specify the maximum time in seconds that you allow the http +transfer operation to take. + @item Related CURL Flags: +CURLOPT_TIMEOUT, CURLOPT_NOSIGNAL + @end enumerate + +@item HTTP.PROXY_SERVER + @enumerate + @item Type: String representing url to access the proxy: + (e.g.http://[username:password@@]host[:port]) + @item Description: +Specify the needed information for accessing a proxy. + @item Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD + @end enumerate + @end itemize -For ESG, the CURL.SSL.CERTIFICATE and CURL.SSL.KEY entries -should have same value, -which is the file path for the certificate produced by MyProxyLogon. -The CURL.SSL.CAPATH entry should be the path to the "certificates" -directory produced by MyProxyLogon. - -Support for other SSL servers can be added by sending a request -to support-netcdf@@unidata.ucar.edu. +The related curl flags line indicates the curl flags modified +by this key. See the libcurl documentation of the curl_easy_setopt() +function for more detail +@uref{http://curl.haxx.se/libcurl/c/curl_easy_setopt.html}. +For ESG, the following entries must be specified: +@itemize +@item HTTP.SSL.VALIDATE +@item HTTP.COOKIEJAR +@item HTTP.SSL.CERTIFICATE +@item HTTP.SSL.KEY +@item HTTP.SSL.CAPATH +@end itemize +Additionally, for ESG, the HTTP.SSL.CERTIFICATE and HTTP.SSL.KEY +entries should have same value, which is the file path for the +certificate produced by MyProxyLogon. The HTTP.SSL.CAPATH entry +should be the path to the "certificates" directory produced by +MyProxyLogon. @node NetCDF Utilities, Units, Structure, Top @chapter NetCDF Utilities diff --git a/oc/curlfunctions.c b/oc/curlfunctions.c index 1665b707f..15801fdee 100644 --- a/oc/curlfunctions.c +++ b/oc/curlfunctions.c @@ -53,6 +53,12 @@ ocset_curl_flags(OCstate* state) OCDBG1(1,"CURLOPT_VERBOSE=%ld",1L); } + if (flags->timeout) { + cstat = curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)flags->timeout); + if (cstat != CURLE_OK) goto fail; + OCDBG1(1,"CURLOPT_TIMEOUT=%ld",1L); + } + /* Following are always set */ cstat = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); OCDBG1(1,"CURLOPT_FOLLOWLOCATION=%ld",1L); diff --git a/oc/ocinternal.c b/oc/ocinternal.c index b3e276758..dd1986576 100644 --- a/oc/ocinternal.c +++ b/oc/ocinternal.c @@ -30,9 +30,8 @@ #define TMPPATH2 "./" #endif -/* Define default rc files */ -#define DODSRC ".dodsrc" -#define OPENDAPRC ".opendap.rc" +/* Define default rc files and aliases*/ +static char* rcfilenames[3] = {".dodsrc",".httprc",NULL}; static int ocextractdds(OCstate*,OCtree*); static char* constraintescape(const char* url); @@ -157,34 +156,37 @@ ocinternalinitialize(void) { char* path = NULL; char* homepath = NULL; + char** alias; FILE* f = NULL; /* locate the configuration files: . first, then $HOME */ - path = (char*)malloc(strlen("./")+strlen(DODSRC)+1); - if(path == NULL) return OC_ENOMEM; - strcpy(path,"./"); - strcat(path,DODSRC); - /* see if file is readable */ - f = fopen(path,"r"); - if(f == NULL) { + for(alias=rcfilenames;*alias;alias++) { + path = (char*)malloc(strlen("./")+strlen(*alias)+1); + if(path == NULL) return OC_ENOMEM; + strcpy(path,"./"); + strcat(path,*alias); + /* see if file is readable */ + f = fopen(path,"r"); + if(f != NULL) break; /* try $HOME */ homepath = getenv("HOME"); if (homepath!= NULL) { - if(path != NULL) free(path); - path = (char*)malloc(strlen(homepath)+1+strlen(DODSRC)+1); - if(path == NULL) return OC_ENOMEM; - strcpy(path,homepath); - strcat(path,"/"); - strcat(path,DODSRC); - f = fopen(path,"r"); + if(path != NULL) free(path); + path = (char*)malloc(strlen(homepath)+1+strlen(*alias)+1); + if(path == NULL) return OC_ENOMEM; + strcpy(path,homepath); + strcat(path,"/"); + strcat(path,*alias); + f = fopen(path,"r"); + if(f != NULL) break; } - } + } if(f == NULL) { - oc_log(LOGWARN,"Cannot find runtime .dodsrc configuration file"); + oc_log(LOGWARN,"Cannot find runtime configuration file"); } else { fclose(f); if(ocdebug > 1) fprintf(stderr, "DODS RC file: %s\n", path); - if(ocdodsrc_read(path) == 0) + if(ocdodsrc_read(*alias,path) == 0) oc_log(LOGERR, "Error parsing %s\n",path); } if(path != NULL) {free(path) ; path = NULL;} @@ -587,7 +589,7 @@ ocsetcurlproperties(OCstate* state) /* process the triple store wrt to this state */ if(ocdodsrc_process(state) != OC_NOERR) { - oc_log(LOGERR,"Malformed .dodsrc"); + oc_log(LOGERR,"Malformed .opendaprc configuration file"); goto fail; } if(state->creds.username == NULL && state->creds.password == NULL) { diff --git a/oc/ocinternal.h b/oc/ocinternal.h index c1ada6ca0..637c9a008 100644 --- a/oc/ocinternal.h +++ b/oc/ocinternal.h @@ -98,6 +98,7 @@ typedef struct OCstate struct OCcurlflags { int compress; int verbose; + int timeout; int followlocation; int maxredirs; char* useragent; diff --git a/oc/rc.c b/oc/rc.c index 393aebde0..25f898c87 100644 --- a/oc/rc.c +++ b/oc/rc.c @@ -24,6 +24,9 @@ #define TRIM(x) rctrimright(rctrimleft((x),TRIMCHARS),TRIMCHARS) +#define HTTPPREFIXDEPRECATED "CURL." +#define HTTPPREFIX "HTTP." + /* the .dodsrc triple store */ struct OCTriplestore* ocdodsrc = NULL; @@ -34,6 +37,8 @@ static char* rctrimleft(char* more, char* trimchars); static void ocdodsrcdump(char* msg, struct OCTriple*, int ntriples); +static char* curllookup(char* suffix,char* url); + /* The Username and password are in the URL if the URL is of the form: * http://:@/.... */ @@ -272,7 +277,7 @@ sorttriplestore(void) /* Create a triple store from a .dodsrc */ int -ocdodsrc_read(char *in_file_name) +ocdodsrc_read(char* basename, char *in_file_name) { char line0[MAXRCLINESIZE]; FILE *in_file = NULL; @@ -289,7 +294,7 @@ ocdodsrc_read(char *in_file_name) in_file = fopen(in_file_name, "r"); /* Open the file to read it */ if (in_file == NULL) { - oc_log(LOGERR, "Could not open the .dodsrc file"); + oc_log(LOGERR, "Could not open configuration file: %s",basename); return OC_EPERM; } @@ -307,16 +312,16 @@ ocdodsrc_read(char *in_file_name) /* trim leading blanks */ line = rctrimleft(line,TRIMCHARS); if(strlen(line) >= MAXRCLINESIZE) { - oc_log(LOGERR, ".dodsrc line too long: %s",line0); + oc_log(LOGERR, "%s line too long: %s",basename,line0); return 0; } /* parse the line */ - ocdodsrc->triples[ocdodsrc->ntriples].url[0] = '\0'; /* assume no url */ + ocdodsrc->triples[ocdodsrc->ntriples].url[0] = '\0'; /*assume no url*/ if(line[0] == LTAG) { char* url = ++line; char* rtag = strchr(line,RTAG); if(rtag == NULL) { - oc_log(LOGERR, "Malformed [url] in .dodsrc entry: %s",line); + oc_log(LOGERR, "Malformed [url] in %s entry: %s",basename,line); continue; } line = rtag + 1; @@ -333,7 +338,7 @@ ocdodsrc_read(char *in_file_name) if(value == NULL) { /* add fake '=1' */ if(strlen(line) + strlen("=1") >= MAXRCLINESIZE) { - oc_log(LOGERR, ".dodsrc entry too long: %s",line); + oc_log(LOGERR, "%s entry too long: %s",basename,line); continue; } strcat(line,"=1"); @@ -357,26 +362,31 @@ ocdodsrc_process(OCstate* state) char* value; char* url = ocuribuild(state->uri,NULL,NULL,0); if(ocdodsrc == NULL) return 0; - value = ocdodsrc_lookup("CURL.DEFLATE",url); + value = curllookup("DEFLATE",url); if(value != NULL) { if(atoi(value)) state->curlflags.compress = 1; if(ocdebug > 0) oc_log(LOGNOTE,"Compression: %ld", state->curlflags.compress); } - if((value = ocdodsrc_lookup("CURL.VERBOSE",url)) != NULL) { + if((value = curllookup("VERBOSE",url)) != NULL) { if(atoi(value)) state->curlflags.verbose = 1; if(ocdebug > 0) oc_log(LOGNOTE,"curl.verbose: %ld", state->curlflags.verbose); } + if((value = curllookup("TIMEOUT",url)) != NULL) { + if(atoi(value)) state->curlflags.timeout = atoi(value); + if(ocdebug > 0) + oc_log(LOGNOTE,"curl.timeout: %ld", state->curlflags.timeout); + } - if((value = ocdodsrc_lookup("CURL.COOKIEFILE",url)) != NULL) { + if((value = curllookup("COOKIEFILE",url)) != NULL) { state->curlflags.cookiefile = strdup(TRIM(value)); if(!state->curlflags.cookiefile) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"COOKIEFILE: %s", state->curlflags.cookiefile); } - if((value = ocdodsrc_lookup("CURL.COOKIEJAR",url)) - || (value = ocdodsrc_lookup("CURL.COOKIE_JAR",url))) { + if((value = curllookup("COOKIEJAR",url)) + || (value = curllookup("COOKIE_JAR",url))) { state->curlflags.cookiejar = strdup(TRIM(value)); if(!state->curlflags.cookiejar) return OC_ENOMEM; if(ocdebug > 0) @@ -391,32 +401,32 @@ ocdodsrc_process(OCstate* state) state->curlflags.cookiefile = strdup(""); } - if((value = ocdodsrc_lookup("CURL.PROXY_SERVER",url)) != NULL) { + if((value = curllookup("PROXY_SERVER",url)) != NULL) { int stat = parseproxy(state,TRIM(value)); if(stat != OC_NOERR) return stat; } - if((value = ocdodsrc_lookup("CURL.SSL.VALIDATE",url)) != NULL) { + if((value = curllookup("SSL.VALIDATE",url)) != NULL) { if(atoi(value)) state->ssl.validate = 1; if(ocdebug > 0) oc_log(LOGNOTE,"CURL.SSL.VALIDATE: %ld", state->ssl.validate); } - if((value = ocdodsrc_lookup("CURL.SSL.CERTIFICATE",url)) != NULL) { + if((value = curllookup("SSL.CERTIFICATE",url)) != NULL) { state->ssl.certificate = strdup(TRIM(value)); if(!state->ssl.certificate) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"CREDENTIALS.SSL.CERTIFICATE: %s", state->ssl.certificate); } - if((value = ocdodsrc_lookup("CURL.SSL.KEY",url)) != NULL) { + if((value = curllookup("SSL.KEY",url)) != NULL) { state->ssl.key = strdup(TRIM(value)); if(!state->ssl.key) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"CREDENTIALS.SSL.KEY: %s", state->ssl.key); } - if((value = ocdodsrc_lookup("CURL.SSL.KEYPASSWORD",url)) != NULL) { + if((value = curllookup("SSL.KEYPASSWORD",url)) != NULL) { state->ssl.keypasswd = strdup(TRIM(value)); if(!state->ssl.keypasswd) return OC_ENOMEM; #ifdef INSECURE @@ -425,28 +435,28 @@ ocdodsrc_process(OCstate* state) #endif } - if((value = ocdodsrc_lookup("CURL.SSL.CAINFO",url)) != NULL) { + if((value = curllookup("SSL.CAINFO",url)) != NULL) { state->ssl.cainfo = strdup(TRIM(value)); if(!state->ssl.cainfo) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"SSL.CAINFO: %s", state->ssl.cainfo); } - if((value = ocdodsrc_lookup("CURL.SSL.CAPATH",url)) != NULL) { + if((value = curllookup("SSL.CAPATH",url)) != NULL) { state->ssl.capath = strdup(TRIM(value)); if(!state->ssl.capath) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"SSL.CAPATH: %s", state->ssl.capath); } - if((value = ocdodsrc_lookup("CURL.CREDENTIALS.USER",url)) != NULL) { + if((value = curllookup("CREDENTIALS.USER",url)) != NULL) { state->creds.username = strdup(TRIM(value)); if(!state->creds.username) return OC_ENOMEM; if(ocdebug > 0) oc_log(LOGNOTE,"CREDENTIALS.USER: %s", state->creds.username); } - if((value = ocdodsrc_lookup("CURL.CREDENTIALS.PASSWORD",url)) != NULL) { + if((value = curllookup("CREDENTIALS.PASSWORD",url)) != NULL) { state->creds.password = strdup(TRIM(value)); if(!state->creds.password) return OC_ENOMEM; } @@ -503,3 +513,20 @@ ocdodsrcdump(char* msg, struct OCTriple* triples, int ntriples) triples[i].value); } } + +/* Isolate the "CURL." prefix to allow changing to something else */ +static char* +curllookup(char* suffix, char* url) +{ + char key[2048]; + char* value = NULL; + strcpy(key,HTTPPREFIX); + strcat(key,suffix); + value = ocdodsrc_lookup(key,url); + if(value == NULL) { + strcpy(key,HTTPPREFIXDEPRECATED); + strcat(key,suffix); + value = ocdodsrc_lookup(key,url); + } + return value; +} diff --git a/oc/rc.h b/oc/rc.h index 362baf932..853ab8272 100644 --- a/oc/rc.h +++ b/oc/rc.h @@ -26,7 +26,7 @@ extern struct OCTriplestore { } triples[MAXRCLINES]; } *ocdodsrc; -extern int ocdodsrc_read(char *in_file_name); +extern int ocdodsrc_read(char* basename,char *in_file_name); extern int ocdodsrc_process(OCstate* state); extern char* ocdodsrc_lookup(char* key, char* url);