openssl/ssl/record/ssl3_buffer.c
Richard Levitte e077455e9e Stop raising ERR_R_MALLOC_FAILURE in most places
Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)
2022-10-05 14:02:03 +02:00

135 lines
3.5 KiB
C

/*
* Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "../ssl_local.h"
#include "record_local.h"
void SSL3_BUFFER_set_data(SSL3_BUFFER *b, const unsigned char *d, size_t n)
{
if (d != NULL)
memcpy(b->buf, d, n);
b->left = n;
b->offset = 0;
}
/*
* Clear the contents of an SSL3_BUFFER but retain any memory allocated. Also
* retains the default_len setting
*/
void SSL3_BUFFER_clear(SSL3_BUFFER *b)
{
b->offset = 0;
b->left = 0;
}
void SSL3_BUFFER_release(SSL3_BUFFER *b)
{
OPENSSL_free(b->buf);
b->buf = NULL;
}
int ssl3_setup_write_buffer(SSL_CONNECTION *s, size_t numwpipes, size_t len)
{
unsigned char *p;
size_t align = 0, headerlen;
SSL3_BUFFER *wb;
size_t currpipe;
/*
* TODO(RECLAYER): Eventually this function can be removed once everything
* is moved to the write record layer.
*/
if (!SSL_CONNECTION_IS_DTLS(s))
return 1;
s->rlayer.numwpipes = numwpipes;
if (len == 0) {
if (SSL_CONNECTION_IS_DTLS(s))
headerlen = DTLS1_RT_HEADER_LENGTH + 1;
else
headerlen = SSL3_RT_HEADER_LENGTH;
#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0
align = SSL3_ALIGN_PAYLOAD - 1;
#endif
len = ssl_get_max_send_fragment(s)
+ SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD + headerlen + align;
#ifndef OPENSSL_NO_COMP
if (ssl_allow_compression(s))
len += SSL3_RT_MAX_COMPRESSED_OVERHEAD;
#endif
if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS))
len += headerlen + align + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD;
}
wb = RECORD_LAYER_get_wbuf(&s->rlayer);
for (currpipe = 0; currpipe < numwpipes; currpipe++) {
SSL3_BUFFER *thiswb = &wb[currpipe];
if (thiswb->len != len) {
OPENSSL_free(thiswb->buf);
thiswb->buf = NULL; /* force reallocation */
}
if (thiswb->buf == NULL) {
if (s->wbio == NULL || !BIO_get_ktls_send(s->wbio)) {
p = OPENSSL_malloc(len);
if (p == NULL) {
s->rlayer.numwpipes = currpipe;
/*
* We've got a malloc failure, and we're still initialising
* buffers. We assume we're so doomed that we won't even be able
* to send an alert.
*/
SSLfatal(s, SSL_AD_NO_ALERT, ERR_R_CRYPTO_LIB);
return 0;
}
} else {
p = NULL;
}
memset(thiswb, 0, sizeof(SSL3_BUFFER));
thiswb->buf = p;
thiswb->len = len;
}
}
return 1;
}
int ssl3_setup_buffers(SSL_CONNECTION *s)
{
if (!ssl3_setup_write_buffer(s, 1, 0)) {
/* SSLfatal() already called */
return 0;
}
return 1;
}
int ssl3_release_write_buffer(SSL_CONNECTION *s)
{
SSL3_BUFFER *wb;
size_t pipes;
pipes = s->rlayer.numwpipes;
while (pipes > 0) {
wb = &RECORD_LAYER_get_wbuf(&s->rlayer)[pipes - 1];
if (SSL3_BUFFER_is_app_buffer(wb))
SSL3_BUFFER_set_app_buffer(wb, 0);
else
OPENSSL_free(wb->buf);
wb->buf = NULL;
pipes--;
}
s->rlayer.numwpipes = 0;
return 1;
}