mirror/curl
2
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2024-11-27 05:50:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support Metalink.

Browse Source 
This change adds experimental Metalink support to curl.
To enable Metalink support, run configure with --with-libmetalink.
To feed Metalink file to curl, use --metalink option like this:

  $ curl -O --metalink foo.metalink

We use libmetalink to parse Metalink files.
This commit is contained in:
Tatsuhiro Tsujikawa 2012-04-26 22:59:52 +09:00 committed by Daniel Stenberg
parent efb8471a69
commit b5fdbe848b
9 changed files with 1225 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
configure.ac
View File

@ -164,6 +164,7 @@ curl_verbose_msg="enabled (--disable-verbose)"
curl_rtsp_msg="no (--enable-rtsp)" curl_rtsp_msg="no (--enable-rtsp)"
curl_rtmp_msg="no (--with-librtmp)" curl_rtmp_msg="no (--with-librtmp)"
init_ssl_msg=${curl_ssl_msg} init_ssl_msg=${curl_ssl_msg}
curl_metalink_msg="no (--with-libmetalink)"
dnl dnl
dnl Save anything in $LIBS for later dnl Save anything in $LIBS for later
@ -2216,6 +2217,63 @@ dnl **********************************************************************
CURL_CHECK_CA_BUNDLE CURL_CHECK_CA_BUNDLE
dnl **********************************************************************
dnl Check for libmetalink
dnl **********************************************************************
OPT_LIBMETALINK=no
AC_ARG_WITH(libmetalink,dnl
AC_HELP_STRING([--with-libmetalink=PATH],[where to look for libmetalink, PATH points to the installation root])
AC_HELP_STRING([--without-libmetalink], [disable libmetalink detection]),
OPT_LIBMETALINK=$withval)
if test X"$OPT_LIBMETALINK" != Xno; then
addlib=""
addld=""
addcflags=""
version=""
libmetalinklib=""
PKGTEST="no"
if test "x$OPT_LIBMETALINK" = "xyes"; then
dnl this is with no partiular path given
PKGTEST="yes"
CURL_CHECK_PKGCONFIG(libmetalink)
else
dnl When particular path is given, set PKG_CONFIG_LIBDIR using the path.
LIBMETALINK_PCDIR="$OPT_LIBMETALINK/lib/pkgconfig"
AC_MSG_NOTICE([PKG_CONFIG_LIBDIR will be set to "$LIBMETALINK_PCDIR"])
if test -f "$LIBMETALINK_PCDIR/libmetalink.pc"; then
PKGTEST="yes"
fi
if test "$PKGTEST" = "yes"; then
CURL_CHECK_PKGCONFIG(libmetalink, [$LIBMETALINK_PCDIR])
fi
fi
if test "$PKGTEST" = "yes" && test "$PKGCONFIG" != "no"; then
addlib=`$PKGCONFIG --libs-only-l libmetalink`
addld=`$PKGCONFIG --libs-only-L libmetalink`
addcflags=`$PKGCONFIG --cflags-only-I libmetalink`
version=`$PKGCONFIG --modversion libmetalink`
libmetalinklib=`echo $addld | $SED -e 's/-L//'`
fi
if test -n "$addlib"; then
AC_MSG_NOTICE([detected libmetalink version $version])
curl_metalink_msg="enabled"
LIBMETALINK_LIBS=$addlib
LIBMETALINK_LDFLAGS=$addld
LIBMETALINK_CFLAGS=$addcflags
AC_SUBST([LIBMETALINK_LIBS])
AC_SUBST([LIBMETALINK_LDFLAGS])
AC_SUBST([LIBMETALINK_CFLAGS])
AC_DEFINE(HAVE_LIBMETALINK, 1, [if libmetalink is available])
fi
fi
AM_CONDITIONAL([HAVE_LIBMETALINK], [test -n "x$LIBMETALINK_LIBS"])
dnl ********************************************************************** dnl **********************************************************************
dnl Check for the presence of LIBSSH2 libraries and headers dnl Check for the presence of LIBSSH2 libraries and headers
dnl ********************************************************************** dnl **********************************************************************
@ -3319,6 +3377,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
LDAPS support: ${curl_ldaps_msg} LDAPS support: ${curl_ldaps_msg}
RTSP support: ${curl_rtsp_msg} RTSP support: ${curl_rtsp_msg}
RTMP support: ${curl_rtmp_msg} RTMP support: ${curl_rtmp_msg}
metalink support: ${curl_metalink_msg}
Protocols: ${SUPPORT_PROTOCOLS} Protocols: ${SUPPORT_PROTOCOLS}
]) ])

1
src/Makefile-metalink.inc Normal file
View File

@ -0,0 +1 @@
curl_SOURCES += tool_metalink.c tool_metalink.h

8
src/Makefile.am
View File

@ -50,11 +50,15 @@ AM_CPPFLAGS = -DCURL_STATICLIB
endif endif
include Makefile.inc include Makefile.inc
if HAVE_LIBMETALINK
include Makefile-metalink.inc
endif # HAVE_LIBMETALINK
# This might hold -Werror # This might hold -Werror
CFLAGS += @CURL_CFLAG_EXTRAS@ CFLAGS += @CURL_CFLAG_EXTRAS@ @LIBMETALINK_CFLAGS@
curl_LDADD = $(top_builddir)/lib/libcurl.la @CURL_LIBS@ curl_LDADD = $(top_builddir)/lib/libcurl.la @CURL_LIBS@ @LIBMETALINK_LIBS@
curl_LDFLAGS = @LIBMETALINK_LDFLAGS@
curl_DEPENDENCIES = $(top_builddir)/lib/libcurl.la curl_DEPENDENCIES = $(top_builddir)/lib/libcurl.la
BUILT_SOURCES = hugehelp.c BUILT_SOURCES = hugehelp.c
CLEANFILES = hugehelp.c CLEANFILES = hugehelp.c

11
src/tool_cfgable.h
View File

@ -25,6 +25,10 @@
#include "tool_sdecls.h" #include "tool_sdecls.h"
#ifdef HAVE_LIBMETALINK
#include "tool_metalink.h"
#endif /* HAVE_LIBMETALINK */
struct Configurable { struct Configurable {
CURL *easy; /* once we have one, we keep it here */ CURL *easy; /* once we have one, we keep it here */
bool remote_time; bool remote_time;
@ -198,6 +202,13 @@ struct Configurable {
bool xattr; /* store metadata in extended attributes */ bool xattr; /* store metadata in extended attributes */
long gssapi_delegation; long gssapi_delegation;
bool ssl_allow_beast; /* allow this SSL vulnerability */ bool ssl_allow_beast; /* allow this SSL vulnerability */
#ifdef HAVE_LIBMETALINK
struct metalinkfile *metalinkfile_list; /* point to the first node */
struct metalinkfile *metalinkfile_last; /* point to the last/current node */
struct metalink *metalink_list; /* point to the first node */
struct metalink *metalink_last; /* point to the last/current node */
#endif /* HAVE_LIBMETALINK */
}; /* struct Configurable */ }; /* struct Configurable */
void free_config_fields(struct Configurable *config); void free_config_fields(struct Configurable *config);

73
src/tool_getparam.c
View File

@ -44,6 +44,10 @@
#include "tool_parsecfg.h" #include "tool_parsecfg.h"
#include "tool_version.h" #include "tool_version.h"
#ifdef HAVE_LIBMETALINK
# include "tool_metalink.h"
#endif /* HAVE_LIBMETALINK */
#include "memdebug.h" /* keep this as LAST include */ #include "memdebug.h" /* keep this as LAST include */
#ifdef MSDOS #ifdef MSDOS
@ -170,6 +174,9 @@ static const struct LongShort aliases[]= {
{"$G", "delegation", TRUE}, {"$G", "delegation", TRUE},
{"$H", "mail-auth", TRUE}, {"$H", "mail-auth", TRUE},
{"$I", "post303", FALSE}, {"$I", "post303", FALSE},
#ifdef HAVE_LIBMETALINK
{"$J", "metalink", TRUE},
#endif /* HAVE_LIBMETALINK */
{"0", "http1.0", FALSE}, {"0", "http1.0", FALSE},
{"1", "tlsv1", FALSE}, {"1", "tlsv1", FALSE},
{"2", "sslv2", FALSE}, {"2", "sslv2", FALSE},
@ -819,6 +826,72 @@ ParameterError getparameter(char *flag, /* f or -long-flag */
case 'H': /* --mail-auth */ case 'H': /* --mail-auth */
GetStr(&config->mail_auth, nextarg); GetStr(&config->mail_auth, nextarg);
break; break;
#ifdef HAVE_LIBMETALINK
case 'J': /* --metalink */
{
metalink_error_t r;
metalink_t* metalink;
metalink_file_t **files;
struct metalink *ml;
r = metalink_parse_file(nextarg, &metalink);
if(r != 0) {
fprintf(stderr, "ERROR: code=%d\n", r);
exit(EXIT_FAILURE);
}
ml = new_metalink(metalink);
if(config->metalink_list) {
config->metalink_last->next = ml;
config->metalink_last = ml;
}
else {
config->metalink_list = config->metalink_last = ml;
}
for(files = metalink->files; *files; ++files) {
struct getout *url;
/* Skip an entry which has no resource. */
if(!(*files)->resources[0]) continue;
if(config->url_get ||
((config->url_get = config->url_list) != NULL)) {
/* there's a node here, if it already is filled-in continue to
find an "empty" node */
while(config->url_get && (config->url_get->flags & GETOUT_URL))
config->url_get = config->url_get->next;
}
/* now there might or might not be an available node to fill in! */
if(config->url_get)
/* existing node */
url = config->url_get;
else
/* there was no free node, create one! */
url=new_getout(config);
if(url) {
struct metalinkfile *mlfile;
/* Set name as url */
GetStr(&url->url, (*files)->name);
/* set flag metalink here */
url->flags |= GETOUT_URL | GETOUT_METALINK;
mlfile = new_metalinkfile(*files);
if(config->metalinkfile_list) {
config->metalinkfile_last->next = mlfile;
config->metalinkfile_last = mlfile;
}
else {
config->metalinkfile_list = config->metalinkfile_last = mlfile;
}
}
}
break;
}
#endif /* HAVE_LIBMETALINK */
} }
break; break;
case '#': /* --progress-bar */ case '#': /* --progress-bar */

990
src/tool_metalink.c Normal file
View File

@ -0,0 +1,990 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include "tool_setup.h"
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_UTIME_H
# include <utime.h>
#elif defined(HAVE_SYS_UTIME_H)
# include <sys/utime.h>
#endif
#define ENABLE_CURLX_PRINTF
/* use our own printf() functions */
#include "curlx.h"
#include "tool_metalink.h"
#include "tool_binmode.h"
#include "tool_cb_prg.h"
#include "tool_cb_wrt.h"
#include "tool_cb_rea.h"
#include "tool_cb_see.h"
#include "tool_cb_dbg.h"
#include "tool_dirhie.h"
#include "tool_util.h"
#include "tool_main.h"
#include "tool_msgs.h"
#include "tool_setopt.h"
#include "tool_sleep.h"
#include "tool_writeout.h"
#include "tool_libinfo.h"
#include "tool_homedir.h"
#include "tool_cb_hdr.h"
#include "memdebug.h" /* keep this as LAST include */
static int curl_truncate_file(struct Configurable *config,
struct OutStruct outs)
{
int res = 0;
if(outs.bytes && outs.filename) {
/* We have written data to a output file, we truncate file
*/
if(!config->mute)
fprintf(config->errors, "Throwing away %"
CURL_FORMAT_CURL_OFF_T " bytes\n",
outs.bytes);
fflush(outs.stream);
/* truncate file at the position where we started appending */
#ifdef HAVE_FTRUNCATE
if(ftruncate( fileno(outs.stream), outs.init)) {
/* when truncate fails, we can't just append as then we'll
create something strange, bail out */
if(!config->mute)
fprintf(config->errors,
"failed to truncate, exiting\n");
res = CURLE_WRITE_ERROR;
assert(0);
/* goto quit_urls; */
}
/* now seek to the end of the file, the position where we
just truncated the file in a large file-safe way */
fseek(outs.stream, 0, SEEK_END);
#else
/* ftruncate is not available, so just reposition the file
to the location we would have truncated it. This won't
work properly with large files on 32-bit systems, but
most of those will have ftruncate. */
fseek(outs.stream, (long)outs.init, SEEK_SET);
#endif
outs.bytes = 0; /* clear for next round */
}
return res;
}
struct metalinkfile *new_metalinkfile(metalink_file_t *metalinkfile) {
struct metalinkfile *f;
f = (struct metalinkfile*)malloc(sizeof(struct metalinkfile));
f->file = metalinkfile;
f->next = NULL;
return f;
}
struct metalink *new_metalink(metalink_t *metalink) {
struct metalink *ml;
ml = (struct metalink*)malloc(sizeof(struct metalink));
ml->metalink = metalink;
ml->next = NULL;
return ml;
}
int operatemetalink(CURL *curl,
struct getout *urlnode,
long retry_sleep_default,
struct OutStruct outs,
struct OutStruct heads,
char *outfiles,
struct Configurable *config)
{
long retry_numretries;
int infd = STDIN_FILENO;
bool infdopen;
char *outfile;
struct timeval retrystart;
struct metalinkfile *mlfile;
int res = 0;
char *filename;
metalink_resource_t **mlres;
char errorbuffer[CURL_ERROR_SIZE];
struct ProgressData progressbar;
long retry_sleep;
struct InStruct input;
curl_off_t uploadfilesize; /* -1 means unknown */
char *uploadfile=NULL; /* a single file, never a glob */
uploadfilesize=-1;
mlfile = config->metalinkfile_last;
config->metalinkfile_last = config->metalinkfile_last->next;
filename = strdup(mlfile->file->name);
outfile = outfiles?strdup(outfiles):NULL;
if((urlnode->flags & GETOUT_USEREMOTE) ||
(outfile && !curlx_strequal("-", outfile)) ) {
/*
* We have specified a file name to store the result in, or we have
* decided we want to use the name attribute of file element in Metalink
* XML.
*/
if(!outfile) {
/* Find and get file name */
char * pc;
pc = strrchr(filename, '/');
if(pc) {
/* duplicate the string beyond the slash */
pc++;
outfile = *pc ? strdup(pc): NULL;
}
else {
outfile = strdup(filename);
}
if(!outfile || !*outfile) {
helpf(config->errors, "Metalink file[@name] has no length!\n");
res = CURLE_WRITE_ERROR;
free(filename);
/* break; */
return 1;
}
#if defined(MSDOS)
{
/* This is for DOS, and then we do some major replacing of
bad characters in the file name before using it */
char file1[PATH_MAX];
if(strlen(outfile) >= PATH_MAX)
outfile[PATH_MAX-1]=0; /* cut it */
strcpy(file1, msdosify(outfile));
free(outfile);
outfile = strdup(rename_if_dos_device_name(file1));
if(!outfile) {
res = CURLE_OUT_OF_MEMORY;
break;
}
}
#endif /* MSDOS */
}
/* Create the directory hierarchy, if not pre-existant to a multiple
file output call */
if(config->create_dirs &&
(CURLE_WRITE_ERROR == create_dir_hierarchy(outfile, config->errors)))
return CURLE_WRITE_ERROR;
if(config->resume_from_current) {
/* We're told to continue from where we are now. Get the
size of the file as it is now and open it for append instead */
struct_stat fileinfo;
/* VMS -- Danger, the filesize is only valid for stream files */
if(0 == stat(outfile, &fileinfo))
/* set offset to current file size: */
config->resume_from = fileinfo.st_size;
else
/* let offset be 0 */
config->resume_from = 0;
}
outs.filename = outfile;
outs.s_isreg = TRUE;
if(config->resume_from) {
outs.init = config->resume_from;
/* open file for output: */
outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");
if(!outs.stream) {
helpf(config->errors, "Can't open '%s'!\n", outfile);
return CURLE_WRITE_ERROR;
}
}
else {
outs.stream = NULL; /* open when needed */
}
}
infdopen=FALSE;
if(!config->errors)
config->errors = stderr;
if(!outfile && !config->use_ascii) {
/* We get the output to stdout and we have not got the ASCII/text
flag, then set stdout to be binary */
set_binmode(stdout);
}
/* Loop though all resources in Metalink */
for(mlres = mlfile->file->resources; *mlres; ++mlres) {
int try_next_res = 0;
if(config->tcp_nodelay)
my_setopt(curl, CURLOPT_TCP_NODELAY, 1);
/* where to store */
my_setopt(curl, CURLOPT_WRITEDATA, &outs);
/* what call to write */
my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb);
/* for uploads */
input.fd = infd;
input.config = config;
/* Note that if CURLOPT_READFUNCTION is fread (the default), then
* lib/telnet.c will Curl_poll() on the input file descriptor
* rather then calling the READFUNCTION at regular intervals.
* The circumstances in which it is preferable to enable this
* behaviour, by omitting to set the READFUNCTION & READDATA options,
* have not been determined.
*/
my_setopt(curl, CURLOPT_READDATA, &input);
/* what call to read */
my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb);
/* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what
CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */
my_setopt(curl, CURLOPT_SEEKDATA, &input);
my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb);
if(config->recvpersecond)
/* tell libcurl to use a smaller sized buffer as it allows us to
make better sleeps! 7.9.9 stuff! */
my_setopt(curl, CURLOPT_BUFFERSIZE, config->recvpersecond);
/* size of uploaded file: */
if(uploadfilesize != -1)
my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
my_setopt(curl, CURLOPT_URL, (*mlres)->url); /* what to fetch */
my_setopt(curl, CURLOPT_NOPROGRESS, config->noprogress);
if(config->no_body) {
my_setopt(curl, CURLOPT_NOBODY, 1);
my_setopt(curl, CURLOPT_HEADER, 1);
}
else
my_setopt(curl, CURLOPT_HEADER, config->include_headers);
#if !defined(CURL_DISABLE_PROXY)
{
/* TODO: Make this a run-time check instead of compile-time one. */
my_setopt_str(curl, CURLOPT_PROXY, config->proxy);
my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd);
/* new in libcurl 7.3 */
my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel);
/* new in libcurl 7.5 */
if(config->proxy)
my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver);
/* new in libcurl 7.10 */
if(config->socksproxy) {
my_setopt_str(curl, CURLOPT_PROXY, config->socksproxy);
my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->socksver);
}
/* new in libcurl 7.10.6 */
if(config->proxyanyauth)
my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
(long) CURLAUTH_ANY);
else if(config->proxynegotiate)
my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
(long) CURLAUTH_GSSNEGOTIATE);
else if(config->proxyntlm)
my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
(long) CURLAUTH_NTLM);
else if(config->proxydigest)
my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
(long) CURLAUTH_DIGEST);
else if(config->proxybasic)
my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
(long) CURLAUTH_BASIC);
/* new in libcurl 7.19.4 */
my_setopt(curl, CURLOPT_NOPROXY, config->noproxy);
}
#endif
my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror);
my_setopt(curl, CURLOPT_UPLOAD, uploadfile?TRUE:FALSE);
my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly);
my_setopt(curl, CURLOPT_APPEND, config->ftp_append);
if(config->netrc_opt)
my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
else if(config->netrc || config->netrc_file)
my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_REQUIRED);
else
my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED);
if(config->netrc_file)
my_setopt(curl, CURLOPT_NETRC_FILE, config->netrc_file);
my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii);
my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd);
my_setopt_str(curl, CURLOPT_RANGE, config->range);
my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer);
my_setopt(curl, CURLOPT_TIMEOUT, config->timeout);
if(built_in_protos & CURLPROTO_HTTP) {
long postRedir = 0;
my_setopt(curl, CURLOPT_FOLLOWLOCATION,
config->followlocation);
my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH,
config->unrestricted_auth);
switch(config->httpreq) {
case HTTPREQ_SIMPLEPOST:
my_setopt_str(curl, CURLOPT_POSTFIELDS,
config->postfields);
my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
config->postfieldsize);
break;
case HTTPREQ_POST:
my_setopt_httppost(curl, CURLOPT_HTTPPOST, config->httppost);
break;
default:
break;
}
my_setopt_str(curl, CURLOPT_REFERER, config->referer);
my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer);
my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent);
my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers);
/* new in libcurl 7.5 */
my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs);
/* new in libcurl 7.9.1 */
if(config->httpversion)
my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion);
/* new in libcurl 7.10.6 (default is Basic) */
if(config->authtype)
my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long) config->authtype);
/* curl 7.19.1 (the 301 version existed in 7.18.2),
303 was added in 7.26.0 */
if(config->post301)
postRedir |= CURL_REDIR_POST_301;
if(config->post302)
postRedir |= CURL_REDIR_POST_302;
if(config->post303)
postRedir |= CURL_REDIR_POST_303;
my_setopt(curl, CURLOPT_POSTREDIR, postRedir);
/* new in libcurl 7.21.6 */
if(config->encoding)
my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, "");
/* new in libcurl 7.21.6 */
if(config->tr_encoding)
my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1);
} /* (built_in_protos & CURLPROTO_HTTP) */
my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);
my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT,
config->low_speed_limit);
my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE,
config->sendpersecond);
my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE,
config->recvpersecond);
my_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
config->use_resume?config->resume_from:0);
my_setopt(curl, CURLOPT_SSLCERT, config->cert);
my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type);
my_setopt(curl, CURLOPT_SSLKEY, config->key);
my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type);
my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd);
if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
/* SSH and SSL private key uses same command-line option */
/* new in libcurl 7.16.1 */
my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key);
/* new in libcurl 7.16.1 */
my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey);
/* new in libcurl 7.17.1: SSH host key md5 checking allows us
to fail if we are not talking to who we think we should */
my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5,
config->hostpubmd5);
}
if(config->cacert)
my_setopt_str(curl, CURLOPT_CAINFO, config->cacert);
if(config->capath)
my_setopt_str(curl, CURLOPT_CAPATH, config->capath);
if(config->crlfile)
my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile);
if(curlinfo->features & CURL_VERSION_SSL) {
if(config->insecure_ok) {
my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1L);
}
else {
my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
/* libcurl default is strict verifyhost -> 2L */
/* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */
}
}
if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
if(!config->insecure_ok) {
char *home;
char *file;
res = CURLE_OUT_OF_MEMORY;
home = homedir();
if(home) {
file = aprintf("%s/%sssh/known_hosts", home, DOT_CHAR);
if(file) {
/* new in curl 7.19.6 */
res = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file);
curl_free(file);
if(res == CURLE_UNKNOWN_OPTION)
/* libssh2 version older than 1.1.1 */
res = CURLE_OK;
}
Curl_safefree(home);
}
if(res)
goto show_error;
}
}
if(config->no_body || config->remote_time) {
/* no body or use remote time */
my_setopt(curl, CURLOPT_FILETIME, TRUE);
}
my_setopt(curl, CURLOPT_CRLF, config->crlf);
my_setopt_slist(curl, CURLOPT_QUOTE, config->quote);
my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote);
my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote);
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES)
{
/* TODO: Make this a run-time check instead of compile-time one. */
if(config->cookie)
my_setopt_str(curl, CURLOPT_COOKIE, config->cookie);
if(config->cookiefile)
my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile);
/* new in libcurl 7.9 */
if(config->cookiejar)
my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar);
/* new in libcurl 7.9.7 */
my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession);
}
#endif
my_setopt_enum(curl, CURLOPT_SSLVERSION, config->ssl_version);
my_setopt_enum(curl, CURLOPT_TIMECONDITION, config->timecond);
my_setopt(curl, CURLOPT_TIMEVALUE, config->condtime);
my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest);
my_setopt(curl, CURLOPT_STDERR, config->errors);
/* three new ones in libcurl 7.3: */
my_setopt_str(curl, CURLOPT_INTERFACE, config->iface);
my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel);
progressbarinit(&progressbar, config);
if((config->progressmode == CURL_PROGRESS_BAR) &&
!config->noprogress && !config->mute) {
/* we want the alternative style, then we have to implement it
ourselves! */
my_setopt(curl, CURLOPT_PROGRESSFUNCTION, tool_progress_cb);
my_setopt(curl, CURLOPT_PROGRESSDATA, &progressbar);
}
/* new in libcurl 7.6.2: */
my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options);
/* new in libcurl 7.7: */
my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file);
my_setopt(curl, CURLOPT_EGDSOCKET, config->egd_file);
my_setopt(curl, CURLOPT_CONNECTTIMEOUT, config->connecttimeout);
if(config->cipher_list)
my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list);
/* new in libcurl 7.9.2: */
if(config->disable_epsv)
/* disable it */
my_setopt(curl, CURLOPT_FTP_USE_EPSV, FALSE);
/* new in libcurl 7.10.5 */
if(config->disable_eprt)
/* disable it */
my_setopt(curl, CURLOPT_FTP_USE_EPRT, FALSE);
if(config->tracetype != TRACE_NONE) {
my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb);
my_setopt(curl, CURLOPT_DEBUGDATA, config);
my_setopt(curl, CURLOPT_VERBOSE, TRUE);
}
/* new in curl 7.9.3 */
if(config->engine) {
res = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
if(res)
goto show_error;
my_setopt(curl, CURLOPT_SSLENGINE_DEFAULT, 1);
}
/* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */
my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS,
config->ftp_create_dirs);
/* new in curl 7.10.8 */
if(config->max_filesize)
my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
config->max_filesize);
if(4 == config->ip_version)
my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
else if(6 == config->ip_version)
my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
else
my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER);
/* new in curl 7.15.5 */
if(config->ftp_ssl_reqd)
my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL);
/* new in curl 7.11.0 */
else if(config->ftp_ssl)
my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
/* new in curl 7.16.0 */
else if(config->ftp_ssl_control)
my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_CONTROL);
/* new in curl 7.16.1 */
if(config->ftp_ssl_ccc)
my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC, config->ftp_ssl_ccc_mode);
#if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI)
{
/* TODO: Make this a run-time check instead of compile-time one. */
/* new in curl 7.19.4 */
if(config->socks5_gssapi_service)
my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_SERVICE,
config->socks5_gssapi_service);
/* new in curl 7.19.4 */
if(config->socks5_gssapi_nec)
my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC,
config->socks5_gssapi_nec);
}
#endif
/* curl 7.13.0 */
my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl);
/* curl 7.14.2 */
my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip);
/* curl 7.15.1 */
my_setopt(curl, CURLOPT_FTP_FILEMETHOD, config->ftp_filemethod);
/* curl 7.15.2 */
if(config->localport) {
my_setopt(curl, CURLOPT_LOCALPORT, config->localport);
my_setopt_str(curl, CURLOPT_LOCALPORTRANGE,
config->localportrange);
}
/* curl 7.15.5 */
my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER,
config->ftp_alternative_to_user);
/* curl 7.16.0 */
if(config->disable_sessionid)
my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE,
!config->disable_sessionid);
/* curl 7.16.2 */
if(config->raw) {
my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE);
my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE);
}
/* curl 7.17.1 */
if(!config->nokeepalive) {
my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
if(config->alivetime != 0) {
#if !defined(TCP_KEEPIDLE) || !defined(TCP_KEEPINTVL)
warnf(config, "Keep-alive functionality somewhat crippled due to "
"missing support in your operating system!\n");
#endif
my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime);
my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime);
}
}
else
my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L);
/* curl 7.20.0 */
if(config->tftp_blksize)
my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize);
if(config->mail_from)
my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from);
if(config->mail_rcpt)
my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt);
/* curl 7.20.x */
if(config->ftp_pret)
my_setopt(curl, CURLOPT_FTP_USE_PRET, TRUE);
if(config->proto_present)
my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto);
if(config->proto_redir_present)
my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir);
if((urlnode->flags & GETOUT_USEREMOTE)
&& config->content_disposition) {
my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb);
my_setopt(curl, CURLOPT_HEADERDATA, &outs);
}
else {
/* if HEADERFUNCTION was set to something in the previous loop, it
is important that we set it (back) to NULL now */
my_setopt(curl, CURLOPT_HEADERFUNCTION, NULL);
my_setopt(curl, CURLOPT_HEADERDATA, config->headerfile?&heads:NULL);
}
if(config->resolve)
/* new in 7.21.3 */
my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve);
/* new in 7.21.4 */
if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) {
if(config->tls_username)
my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME,
config->tls_username);
if(config->tls_password)
my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD,
config->tls_password);
if(config->tls_authtype)
my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE,
config->tls_authtype);
}
/* new in 7.22.0 */
if(config->gssapi_delegation)
my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION,
config->gssapi_delegation);
/* new in 7.25.0 */
if(config->ssl_allow_beast)
my_setopt(curl, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_ALLOW_BEAST);
if(config->mail_auth)
my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth);
/* initialize retry vars for loop below */
retry_sleep_default = (config->retry_delay) ?
config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
retry_numretries = config->req_retry;
retry_sleep = retry_sleep_default; /* ms */
retrystart = tvnow();
#ifndef CURL_DISABLE_LIBCURL_OPTION
res = easysrc_perform();
if(res) {
goto show_error;
}
#endif
for(;;) {
res = curl_easy_perform(curl);
/* if retry-max-time is non-zero, make sure we haven't exceeded the
time */
if(retry_numretries &&
(!config->retry_maxtime ||
(tvdiff(tvnow(), retrystart) < config->retry_maxtime*1000)) ) {
enum {
RETRY_NO,
RETRY_TIMEOUT,
RETRY_HTTP,
RETRY_FTP,
RETRY_LAST /* not used */
} retry = RETRY_NO;
long response;
if(CURLE_OPERATION_TIMEDOUT == res)
/* retry timeout always */
retry = RETRY_TIMEOUT;
else if(CURLE_OK == res) {
/* Check for HTTP transient errors */
char *this_url=NULL;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &this_url);
if(this_url &&
curlx_strnequal(this_url, "http", 4)) {
/* This was HTTP(S) */
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
switch(response) {
case 500: /* Internal Server Error */
case 502: /* Bad Gateway */
case 503: /* Service Unavailable */
case 504: /* Gateway Timeout */
retry = RETRY_HTTP;
/*
* At this point, we have already written data to the output
* file (or terminal). If we write to a file, we must rewind
* or close/re-open the file so that the next attempt starts
* over from the beginning.
*
* TODO: similar action for the upload case. We might need
* to start over reading from a previous point if we have
* uploaded something when this was returned.
*/
break;
}
}
} /* if CURLE_OK */
else if(CURLE_LOGIN_DENIED == res) {
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
if(response/100 == 4)
/*
* This is typically when the FTP server only allows a certain
* amount of users and we are not one of them. All 4xx codes
* are transient.
*/
retry = RETRY_FTP;
}
if(retry) {
static const char * const m[]={NULL,
"timeout",
"HTTP error",
"FTP error"
};
warnf(config, "Transient problem: %s "
"Will retry in %ld seconds. "
"%ld retries left.\n",
m[retry],
retry_sleep/1000,
retry_numretries);
tool_go_sleep(retry_sleep);
retry_numretries--;
if(!config->retry_delay) {
retry_sleep *= 2;
if(retry_sleep > RETRY_SLEEP_MAX)
retry_sleep = RETRY_SLEEP_MAX;
}
curl_truncate_file(config, outs);
continue;
}
} /* if retry_numretries */
else {
/* Metalink: Decide to try the next resource or
not. Basically, we want to try the next resource if
download was not successful. */
long response;
if(CURLE_OPERATION_TIMEDOUT == res) {
try_next_res = 1;
}
else if(CURLE_OK == res) {
char *this_url=NULL;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &this_url);
if(this_url &&
curlx_strnequal(this_url, "http", 4)) {
/* This was HTTP(S) */
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
switch(response) {
case 400:
case 401:
case 402:
case 403:
case 404:
case 405:
case 406:
case 407:
case 408:
case 409:
case 410:
case 411:
case 412:
case 413:
case 414:
case 415:
case 416:
case 417:
try_next_res = 1;
break;
}
}
}
else if(CURLE_LOGIN_DENIED == res) {
if(response/100 == 5)
/*
* For permanent negative return code, try the next resource.
*/
try_next_res = 1;
}
}
/* In all ordinary cases, just break out of loop here */
retry_sleep = retry_sleep_default;
break;
}
if(try_next_res == 1) {
if(mlres+1) {
warnf(config, "The remote server returned negative response. "
"Will try to the next resource.");
}
curl_truncate_file(config, outs);
}
else
break;
}
if((config->progressmode == CURL_PROGRESS_BAR) &&
progressbar.calls) {
/* if the custom progress bar has been displayed, we output a
newline here */
fputs("\n", progressbar.out);
}
if(config->writeout) {
ourWriteOut(curl, &outs, config->writeout);
}
#ifdef USE_ENVIRONMENT
if(config->writeenv)
ourWriteEnv(curl);
#endif
show_error:
#ifdef VMS
if(!config->showerror) {
vms_show = VMSSTS_HIDE;
}
#else
if((res!=CURLE_OK) && config->showerror) {
fprintf(config->errors, "curl: (%d) %s\n", res,
errorbuffer[0]? errorbuffer:
curl_easy_strerror((CURLcode)res));
if(CURLE_SSL_CACERT == res) {
#define CURL_CA_CERT_ERRORMSG1 \
"More details here: http://curl.haxx.se/docs/sslcerts.html\n\n" \
"curl performs SSL certificate verification by default, using a \"bundle\"\n" \
" of Certificate Authority (CA) public keys (CA certs). If the default\n" \
" bundle file isn't adequate, you can specify an alternate file\n" \
" using the --cacert option.\n"
#define CURL_CA_CERT_ERRORMSG2 \
"If this HTTPS server uses a certificate signed by a CA represented in\n" \
" the bundle, the certificate verification probably failed due to a\n" \
" problem with the certificate (it might be expired, or the name might\n" \
" not match the domain name in the URL).\n" \
"If you'd like to turn off curl's verification of the certificate, use\n" \
" the -k (or --insecure) option.\n"
fprintf(config->errors, "%s%s",
CURL_CA_CERT_ERRORMSG1,
CURL_CA_CERT_ERRORMSG2 );
}
}
#endif
if(outfile && !curlx_strequal(outfile, "-") && outs.stream)
fclose(outs.stream);
#ifdef HAVE_UTIME
/* Important that we set the time _after_ the file has been
closed, as is done above here */
if(config->remote_time && outs.filename) {
/* ask libcurl if we got a time. Pretty please */
long filetime;
curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime);
if(filetime >= 0) {
struct utimbuf times;
times.actime = (time_t)filetime;
times.modtime = (time_t)filetime;
utime(outs.filename, &times); /* set the time we got */
}
}
#endif
#ifdef __AMIGA__
/* Set the url as comment for the file. (up to 80 chars are allowed)
*/
if(strlen(url) > 78)
url[79] = '\0';
SetComment( outs.filename, url);
#endif
if(filename)
free(filename);
if(outfile)
free(outfile);
if(infdopen)
close(infd);
if(outfiles)
free(outfiles);
return 0;
}
void clean_metalink(struct Configurable *config)
{
while(config->metalinkfile_list) {
struct metalinkfile *mlfile = config->metalinkfile_list;
config->metalinkfile_list = config->metalinkfile_list->next;
free(mlfile);
}
config->metalinkfile_last = 0;
while(config->metalink_list) {
struct metalink *ml = config->metalink_list;
config->metalink_list = config->metalink_list->next;
metalink_delete(ml->metalink);
free(ml);
}
config->metalink_last = 0;
}

54
src/tool_metalink.h Normal file
View File

@ -0,0 +1,54 @@
#ifndef HEADER_CURL_TOOL_METALINK_H
#define HEADER_CURL_TOOL_METALINK_H
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at http://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#include "tool_setup.h"
#include <metalink/metalink_parser.h>
#include "tool_cfgable.h"
struct metalinkfile {
struct metalinkfile *next;
metalink_file_t *file;
};
struct metalink {
struct metalink *next;
metalink_t* metalink;
};
struct metalinkfile *new_metalinkfile(metalink_file_t *metalinkfile);
struct metalink *new_metalink(metalink_t *metalink);
int operatemetalink(CURL *curl,
struct getout *urlnode,
long retry_sleep_default,
struct OutStruct outs,
struct OutStruct heads,
char *outfiles,
struct Configurable *config);
void clean_metalink(struct Configurable *config);
#endif /* HEADER_CURL_TOOL_METALINK_H */

28
src/tool_operate.c
View File

@ -388,6 +388,11 @@ int operate(struct Configurable *config, int argc, argv_item_t argv[])
} }
} }
#ifdef HAVE_LIBMETALINK
config->metalinkfile_last = config->metalinkfile_list;
config->metalink_last = config->metalink_list;
#endif /* HAVE_LIBMETALINK */
/* /*
** Nested loops start here. ** Nested loops start here.
*/ */
@ -439,6 +444,24 @@ int operate(struct Configurable *config, int argc, argv_item_t argv[])
} }
} }
/* process metalink download in the separate function */
if(urlnode->flags & GETOUT_METALINK) {
struct OutStruct outs;
long retry_sleep_default;
struct getout *nextnode;
retry_sleep_default = (config->retry_delay) ?
config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
/* default output stream is stdout */
memset(&outs, 0, sizeof(struct OutStruct));
outs.stream = stdout;
outs.config = config;
operatemetalink(curl, urlnode, retry_sleep_default, outs, heads,
outfiles, config);
/* move on to the next URL */
continue;
}
/* Here's the loop for uploading multiple files within the same /* Here's the loop for uploading multiple files within the same
single globbed string. If no upload, we enter the loop once anyway. */ single globbed string. If no upload, we enter the loop once anyway. */
for(up = 0 ; up < infilenum; up++) { for(up = 0 ; up < infilenum; up++) {
@ -1608,6 +1631,11 @@ int operate(struct Configurable *config, int argc, argv_item_t argv[])
if(config->errors_fopened && config->errors) if(config->errors_fopened && config->errors)
fclose(config->errors); fclose(config->errors);
#ifdef HAVE_LIBMETALINK
/* Release metalink related resources here */
clean_metalink(config);
#endif /* HAVE_LIBMETALINK */
main_free(); /* cleanup */ main_free(); /* cleanup */
return res; return res;

3
src/tool_sdecls.h
View File

@ -101,6 +101,9 @@ struct getout {
#define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */ #define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */
#define GETOUT_UPLOAD (1<<3) /* if set, -T has been used */ #define GETOUT_UPLOAD (1<<3) /* if set, -T has been used */
#define GETOUT_NOUPLOAD (1<<4) /* if set, -T "" has been used */ #define GETOUT_NOUPLOAD (1<<4) /* if set, -T "" has been used */
#ifdef HAVE_LIBMETALINK
#define GETOUT_METALINK (1<<5) /* set when Metalink download */
#endif /* HAVE_LIBMETALINK */
/* /*