openssl/crypto/rsa/rsa_saos.c
Richard Levitte e077455e9e Stop raising ERR_R_MALLOC_FAILURE in most places
Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)
2022-10-05 14:02:03 +02:00

96 lines
2.6 KiB
C

/*
* Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
/*
* RSA low level APIs are deprecated for public use, but still ok for
* internal use.
*/
#include "internal/deprecated.h"
#include <stdio.h>
#include "internal/cryptlib.h"
#include <openssl/bn.h>
#include <openssl/rsa.h>
#include <openssl/objects.h>
#include <openssl/x509.h>
int RSA_sign_ASN1_OCTET_STRING(int type,
const unsigned char *m, unsigned int m_len,
unsigned char *sigret, unsigned int *siglen,
RSA *rsa)
{
ASN1_OCTET_STRING sig;
int i, j, ret = 1;
unsigned char *p, *s;
sig.type = V_ASN1_OCTET_STRING;
sig.length = m_len;
sig.data = (unsigned char *)m;
i = i2d_ASN1_OCTET_STRING(&sig, NULL);
j = RSA_size(rsa);
if (i > (j - RSA_PKCS1_PADDING_SIZE)) {
ERR_raise(ERR_LIB_RSA, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
return 0;
}
s = OPENSSL_malloc((unsigned int)j + 1);
if (s == NULL)
return 0;
p = s;
i2d_ASN1_OCTET_STRING(&sig, &p);
i = RSA_private_encrypt(i, s, sigret, rsa, RSA_PKCS1_PADDING);
if (i <= 0)
ret = 0;
else
*siglen = i;
OPENSSL_clear_free(s, (unsigned int)j + 1);
return ret;
}
int RSA_verify_ASN1_OCTET_STRING(int dtype,
const unsigned char *m,
unsigned int m_len, unsigned char *sigbuf,
unsigned int siglen, RSA *rsa)
{
int i, ret = 0;
unsigned char *s;
const unsigned char *p;
ASN1_OCTET_STRING *sig = NULL;
if (siglen != (unsigned int)RSA_size(rsa)) {
ERR_raise(ERR_LIB_RSA, RSA_R_WRONG_SIGNATURE_LENGTH);
return 0;
}
s = OPENSSL_malloc((unsigned int)siglen);
if (s == NULL)
goto err;
i = RSA_public_decrypt((int)siglen, sigbuf, s, rsa, RSA_PKCS1_PADDING);
if (i <= 0)
goto err;
p = s;
sig = d2i_ASN1_OCTET_STRING(NULL, &p, (long)i);
if (sig == NULL)
goto err;
if (((unsigned int)sig->length != m_len) ||
(memcmp(m, sig->data, m_len) != 0)) {
ERR_raise(ERR_LIB_RSA, RSA_R_BAD_SIGNATURE);
} else {
ret = 1;
}
err:
ASN1_OCTET_STRING_free(sig);
OPENSSL_clear_free(s, (unsigned int)siglen);
return ret;
}