openssl/crypto/evp/p_seal.c
Richard Levitte e077455e9e Stop raising ERR_R_MALLOC_FAILURE in most places
Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)
2022-10-05 14:02:03 +02:00

87 lines
2.4 KiB
C

/*
* Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include <stdio.h>
#include "internal/cryptlib.h"
#include "internal/provider.h"
#include <openssl/rand.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/objects.h>
#include <openssl/x509.h>
int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
unsigned char **ek, int *ekl, unsigned char *iv,
EVP_PKEY **pubk, int npubk)
{
unsigned char key[EVP_MAX_KEY_LENGTH];
const OSSL_PROVIDER *prov;
OSSL_LIB_CTX *libctx = NULL;
EVP_PKEY_CTX *pctx = NULL;
const EVP_CIPHER *cipher;
int i, len;
int rv = 0;
if (type != NULL) {
EVP_CIPHER_CTX_reset(ctx);
if (!EVP_EncryptInit_ex(ctx, type, NULL, NULL, NULL))
return 0;
}
if ((cipher = EVP_CIPHER_CTX_get0_cipher(ctx)) != NULL
&& (prov = EVP_CIPHER_get0_provider(cipher)) != NULL)
libctx = ossl_provider_libctx(prov);
if ((npubk <= 0) || !pubk)
return 1;
if (EVP_CIPHER_CTX_rand_key(ctx, key) <= 0)
return 0;
len = EVP_CIPHER_CTX_get_iv_length(ctx);
if (len < 0 || RAND_priv_bytes_ex(libctx, iv, len, 0) <= 0)
goto err;
len = EVP_CIPHER_CTX_get_key_length(ctx);
if (len < 0)
goto err;
if (!EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv))
goto err;
for (i = 0; i < npubk; i++) {
size_t keylen = len;
pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
if (pctx == NULL) {
ERR_raise(ERR_LIB_EVP, ERR_R_EVP_LIB);
goto err;
}
if (EVP_PKEY_encrypt_init(pctx) <= 0
|| EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
goto err;
ekl[i] = (int)keylen;
EVP_PKEY_CTX_free(pctx);
}
pctx = NULL;
rv = npubk;
err:
EVP_PKEY_CTX_free(pctx);
OPENSSL_cleanse(key, sizeof(key));
return rv;
}
int EVP_SealFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
{
int i;
i = EVP_EncryptFinal_ex(ctx, out, outl);
if (i)
i = EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, NULL);
return i;
}