From b1d08d295f0e5035f9772654533f3caab64dd8c7 Mon Sep 17 00:00:00 2001 From: Jay Satiro Date: Fri, 1 Oct 2021 13:57:23 -0400 Subject: [PATCH] http: set content length earlier - Make content length (ie download size) accessible to the user in the header callback, but only after all headers have been processed (ie only in the final call to the header callback). Background: For a long time the content length could be retrieved in the header callback via CURLINFO_CONTENT_LENGTH_DOWNLOAD_T as soon as it was parsed by curl. Changes were made in 8a16e54 (precedes 7.79.0) to ignore content length if any transfer encoding is used. A side effect of that was that content length was not set by libcurl until after the header callback was called the final time, because until all headers are processed it cannot be determined if content length is valid. This change keeps the same intention --all headers must be processed-- but now the content length is available before the final call to the header function that indicates all headers have been processed (ie a blank header). Bug: https://github.com/curl/curl/commit/8a16e54#r57374914 Reported-by: sergio-nsk@users.noreply.github.com Co-authored-by: Daniel Stenberg Fixes https://github.com/curl/curl/issues/7804 Closes https://github.com/curl/curl/pull/7803 --- lib/c-hyper.c | 16 +++++++----- lib/http.c | 68 ++++++++++++++++++++++----------------------------- lib/http.h | 2 ++ 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/lib/c-hyper.c b/lib/c-hyper.c index bc3f821359..7ce958adba 100644 --- a/lib/c-hyper.c +++ b/lib/c-hyper.c @@ -299,8 +299,14 @@ static CURLcode status_line(struct Curl_easy *data, */ static CURLcode empty_header(struct Curl_easy *data) { - return hyper_each_header(data, NULL, 0, NULL, 0) ? - CURLE_WRITE_ERROR : CURLE_OK; + CURLcode result = Curl_http_size(data); + if(!result) { + result = hyper_each_header(data, NULL, 0, NULL, 0) ? + CURLE_WRITE_ERROR : CURLE_OK; + if(result) + failf(data, "hyperstream: couldn't pass blank header"); + } + return result; } CURLcode Curl_hyper_stream(struct Curl_easy *data, @@ -443,11 +449,9 @@ CURLcode Curl_hyper_stream(struct Curl_easy *data, break; } - if(empty_header(data)) { - failf(data, "hyperstream: couldn't pass blank header"); - result = CURLE_OUT_OF_MEMORY; + result = empty_header(data); + if(result) break; - } /* Curl_http_auth_act() checks what authentication methods that are * available and decides which one (if any) to use. It will set 'newurl' diff --git a/lib/http.c b/lib/http.c index 8f5457adc8..4777750ec2 100644 --- a/lib/http.c +++ b/lib/http.c @@ -2903,20 +2903,6 @@ CURLcode Curl_http_firstwrite(struct Curl_easy *data, { struct SingleRequest *k = &data->req; - if(data->req.ignore_cl) { - k->size = k->maxdownload = -1; - } - else if(k->size != -1) { - /* We wait until after all headers have been received to set this so that - we know for sure Content-Length is valid. */ - if(data->set.max_filesize && - k->size > data->set.max_filesize) { - failf(data, "Maximum file size exceeded"); - return CURLE_FILESIZE_EXCEEDED; - } - Curl_pgrsSetDownloadSize(data, k->size); - } - if(data->req.newurl) { if(conn->bits.close) { /* Abort after the headers if "follow Location" is set @@ -3787,6 +3773,29 @@ CURLcode Curl_http_statusline(struct Curl_easy *data, return CURLE_OK; } +/* Content-Length must be ignored if any Transfer-Encoding is present in the + response. Refer to RFC 7230 section 3.3.3 and RFC2616 section 4.4. This is + figured out here after all headers have been received but before the final + call to the user's header callback, so that a valid content length can be + retrieved by the user in the final call. */ +CURLcode Curl_http_size(struct Curl_easy *data) +{ + struct SingleRequest *k = &data->req; + if(data->req.ignore_cl || k->chunk) { + k->size = k->maxdownload = -1; + } + else if(k->size != -1) { + if(data->set.max_filesize && + k->size > data->set.max_filesize) { + failf(data, "Maximum file size exceeded"); + return CURLE_FILESIZE_EXCEEDED; + } + Curl_pgrsSetDownloadSize(data, k->size); + k->maxdownload = k->size; + } + return CURLE_OK; +} + /* * Read any HTTP header lines from the server and pass them to the client app. */ @@ -3981,6 +3990,12 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, } } + if(!k->header) { + result = Curl_http_size(data); + if(result) + return result; + } + /* At this point we have some idea about the fate of the connection. If we are closing the connection it may result auth failure. */ #if defined(USE_NTLM) @@ -4137,31 +4152,6 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, reason */ *stop_reading = TRUE; #endif - else { - /* If we know the expected size of this document, we set the - maximum download size to the size of the expected - document or else, we won't know when to stop reading! - - Note that we set the download maximum even if we read a - "Connection: close" header, to make sure that - "Content-Length: 0" still prevents us from attempting to - read the (missing) response-body. - */ - /* According to RFC2616 section 4.4, we MUST ignore - Content-Length: headers if we are now receiving data - using chunked Transfer-Encoding. - */ - if(k->chunk) - k->maxdownload = k->size = -1; - } - if(-1 != k->size) { - /* We do this operation even if no_body is true, since this - data might be retrieved later with curl_easy_getinfo() - and its CURLINFO_CONTENT_LENGTH_DOWNLOAD option. */ - - Curl_pgrsSetDownloadSize(data, k->size); - k->maxdownload = k->size; - } /* If max download size is *zero* (nothing) we already have nothing and can safely return ok now! But for HTTP/2, we'd diff --git a/lib/http.h b/lib/http.h index e4ab466c00..cb5b56faf3 100644 --- a/lib/http.h +++ b/lib/http.h @@ -289,6 +289,8 @@ struct http_conn { #endif }; +CURLcode Curl_http_size(struct Curl_easy *data); + CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, struct connectdata *conn, ssize_t *nread,