tool_operate: reduce errorbuffer allocs

- parallel transfers: only alloc and keep errorbuffers in memory for
  actual "live" transfers and not for the ones in the pending queue

- serial transfers: reuse the same fixed buffer for all transfers, not
  allocated at all.

Closes #9394
This commit is contained in:
Daniel Stenberg 2022-08-30 16:44:12 +02:00
parent c9061f242b
commit 7be53774c4
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
3 changed files with 31 additions and 9 deletions

View File

@ -347,6 +347,9 @@ static void AmigaSetComment(struct per_transfer *per,
#define AmigaSetComment(x,y) Curl_nop_stmt
#endif
/* When doing serial transfers, we use a single fixed error area */
static char global_errorbuffer[CURL_ERROR_SIZE];
/*
* Call this after a transfer has completed.
*/
@ -378,9 +381,9 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
else
#endif
if(!config->synthetic_error && result && global->showerror) {
const char *msg = per->errorbuffer;
fprintf(global->errors, "curl: (%d) %s\n", result,
(per->errorbuffer[0]) ? per->errorbuffer :
curl_easy_strerror(result));
(msg && msg[0]) ? msg : curl_easy_strerror(result));
if(result == CURLE_PEER_FAILED_VERIFICATION)
fputs(CURL_CA_CERT_ERRORMSG, global->errors);
}
@ -656,6 +659,8 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
free(per->this_url);
free(per->outfile);
free(per->uploadfile);
if(global->parallel)
free(per->errorbuffer);
return result;
}
@ -706,7 +711,6 @@ static long url_proto(char *url)
}
/* create the next (singular) transfer */
static CURLcode single_transfer(struct GlobalConfig *global,
struct OperationConfig *config,
CURLSH *share,
@ -1364,7 +1368,10 @@ static CURLcode single_transfer(struct GlobalConfig *global,
my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options);
my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd);
my_setopt_str(curl, CURLOPT_RANGE, config->range);
my_setopt(curl, CURLOPT_ERRORBUFFER, per->errorbuffer);
if(!global->parallel) {
per->errorbuffer = global_errorbuffer;
my_setopt(curl, CURLOPT_ERRORBUFFER, global_errorbuffer);
}
my_setopt(curl, CURLOPT_TIMEOUT_MS, (long)(config->timeout * 1000));
switch(config->httpreq) {
@ -2196,6 +2203,7 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
CURLcode result = CURLE_OK;
CURLMcode mcode;
bool sleeping = FALSE;
char *errorbuf;
*addedp = FALSE;
*morep = FALSE;
if(all_pers < (global->parallel_max*2)) {
@ -2219,6 +2227,13 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
if(result)
return result;
errorbuf = per->errorbuffer;
if(!errorbuf) {
errorbuf = malloc(CURL_ERROR_SIZE);
if(!errorbuf)
return CURLE_OUT_OF_MEMORY;
}
/* parallel connect means that we don't set PIPEWAIT since pipewait
will make libcurl prefer multiplexing */
(void)curl_easy_setopt(per->curl, CURLOPT_PIPEWAIT,
@ -2227,14 +2242,20 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
(void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
(void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per);
(void)curl_easy_setopt(per->curl, CURLOPT_NOPROGRESS, 0L);
(void)curl_easy_setopt(per->curl, CURLOPT_ERRORBUFFER, errorbuf);
mcode = curl_multi_add_handle(multi, per->curl);
if(mcode)
if(mcode) {
free(errorbuf);
return CURLE_OUT_OF_MEMORY;
}
result = create_transfer(global, share, &getadded);
if(result)
if(result) {
free(errorbuf);
return result;
}
per->errorbuffer = errorbuf;
per->added = TRUE;
all_added++;
*addedp = TRUE;

View File

@ -52,7 +52,6 @@ struct per_transfer {
struct HdrCbData hdrcbdata;
long num_headers;
bool was_last_header_empty;
char errorbuffer[CURL_ERROR_SIZE];
bool added; /* set TRUE when added to the multi handle */
time_t startat; /* when doing parallel transfers, this is a retry transfer
@ -72,6 +71,8 @@ struct per_transfer {
/* NULL or malloced */
char *uploadfile;
char *errorbuffer; /* alloced and assigned while this is used for a
transfer */
};
CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]);

View File

@ -195,8 +195,8 @@ static int writeString(FILE *stream, const struct writeoutvar *wovar,
switch(wovar->id) {
case VAR_ERRORMSG:
if(per_result) {
strinfo = per->errorbuffer[0] ? per->errorbuffer :
curl_easy_strerror(per_result);
strinfo = (per->errorbuffer && per->errorbuffer[0]) ?
per->errorbuffer : curl_easy_strerror(per_result);
valid = true;
}
break;