openssl/crypto/o_fopen.c
Richard Levitte e077455e9e Stop raising ERR_R_MALLOC_FAILURE in most places
Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)
2022-10-05 14:02:03 +02:00

125 lines
4.2 KiB
C

/*
* Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
# if defined(__linux) || defined(__sun) || defined(__hpux)
/*
* Following definition aliases fopen to fopen64 on above mentioned
* platforms. This makes it possible to open and sequentially access files
* larger than 2GB from 32-bit application. It does not allow to traverse
* them beyond 2GB with fseek/ftell, but on the other hand *no* 32-bit
* platform permits that, not with fseek/ftell. Not to mention that breaking
* 2GB limit for seeking would require surgery to *our* API. But sequential
* access suffices for practical cases when you can run into large files,
* such as fingerprinting, so we can let API alone. For reference, the list
* of 32-bit platforms which allow for sequential access of large files
* without extra "magic" comprise *BSD, Darwin, IRIX...
*/
# ifndef _FILE_OFFSET_BITS
# define _FILE_OFFSET_BITS 64
# endif
# endif
#include "internal/e_os.h"
#include "internal/cryptlib.h"
#if !defined(OPENSSL_NO_STDIO)
# include <stdio.h>
# ifdef __DJGPP__
# include <unistd.h>
# endif
FILE *openssl_fopen(const char *filename, const char *mode)
{
FILE *file = NULL;
# if defined(_WIN32) && defined(CP_UTF8)
int sz, len_0 = (int)strlen(filename) + 1;
DWORD flags;
/*
* Basically there are three cases to cover: a) filename is
* pure ASCII string; b) actual UTF-8 encoded string and
* c) locale-ized string, i.e. one containing 8-bit
* characters that are meaningful in current system locale.
* If filename is pure ASCII or real UTF-8 encoded string,
* MultiByteToWideChar succeeds and _wfopen works. If
* filename is locale-ized string, chances are that
* MultiByteToWideChar fails reporting
* ERROR_NO_UNICODE_TRANSLATION, in which case we fall
* back to fopen...
*/
if ((sz = MultiByteToWideChar(CP_UTF8, (flags = MB_ERR_INVALID_CHARS),
filename, len_0, NULL, 0)) > 0 ||
(GetLastError() == ERROR_INVALID_FLAGS &&
(sz = MultiByteToWideChar(CP_UTF8, (flags = 0),
filename, len_0, NULL, 0)) > 0)
) {
WCHAR wmode[8];
WCHAR *wfilename = _alloca(sz * sizeof(WCHAR));
if (MultiByteToWideChar(CP_UTF8, flags,
filename, len_0, wfilename, sz) &&
MultiByteToWideChar(CP_UTF8, 0, mode, strlen(mode) + 1,
wmode, OSSL_NELEM(wmode)) &&
(file = _wfopen(wfilename, wmode)) == NULL &&
(errno == ENOENT || errno == EBADF)
) {
/*
* UTF-8 decode succeeded, but no file, filename
* could still have been locale-ized...
*/
file = fopen(filename, mode);
}
} else if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) {
file = fopen(filename, mode);
}
# elif defined(__DJGPP__)
{
char *newname = NULL;
if (pathconf(filename, _PC_NAME_MAX) <= 12) { /* 8.3 file system? */
char *iterator;
char lastchar;
if ((newname = OPENSSL_malloc(strlen(filename) + 1)) == NULL)
return NULL;
for (iterator = newname, lastchar = '\0';
*filename; filename++, iterator++) {
if (lastchar == '/' && filename[0] == '.'
&& filename[1] != '.' && filename[1] != '/') {
/* Leading dots are not permitted in plain DOS. */
*iterator = '_';
} else {
*iterator = *filename;
}
lastchar = *filename;
}
*iterator = '\0';
filename = newname;
}
file = fopen(filename, mode);
OPENSSL_free(newname);
}
# else
file = fopen(filename, mode);
# endif
return file;
}
#else
void *openssl_fopen(const char *filename, const char *mode)
{
return NULL;
}
#endif