Add new filter BIO BIO_f_readbuffer()

This allows BIO_tell() and BIO_seek() to work for BIO's that do
not support these methods. The main use case for this is file/fd BIO's
that use stdin.

This works for stdin taken from input redirection (command < file),
and stdin via pipe (cat file | command).

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14407)

CHANGES.md

@@ -23,6 +23,13 @@ OpenSSL 3.0
 
 ### Changes between 1.1.1 and 3.0 [xx XXX xxxx]
 
+ * Add filter BIO BIO_f_readbuffer() that allows BIO_tell() and BIO_seek() to
+   work on read only BIO source/sinks that do not support these functions.
+   This allows piping or redirection of a file BIO using stdin to be buffered
+   into memory. This is used internally in OSSL_DECODER_from_bio().
+
+   *Shane Lontis*
+
  * OSSL_STORE_INFO_get_type() may now return an additional value. In 1.1.1
    this function would return one of the values OSSL_STORE_INFO_NAME,
    OSSL_STORE_INFO_PKEY, OSSL_STORE_INFO_PARAMS, OSSL_STORE_INFO_CERT or

crypto/bio/bf_readbuff.c

@@ -0,0 +1,268 @@
+/*
+ * Copyright 2021 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/*
+ * This is a read only BIO filter that can be used to add BIO_tell() and
+ * BIO_seek() support to source/sink BIO's (such as a file BIO that uses stdin).
+ * It does this by caching ALL data read from the BIO source/sink into a
+ * resizable memory buffer.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include "bio_local.h"
+#include "internal/cryptlib.h"
+
+#define DEFAULT_BUFFER_SIZE     4096
+
+static int readbuffer_write(BIO *h, const char *buf, int num);
+static int readbuffer_read(BIO *h, char *buf, int size);
+static int readbuffer_puts(BIO *h, const char *str);
+static int readbuffer_gets(BIO *h, char *str, int size);
+static long readbuffer_ctrl(BIO *h, int cmd, long arg1, void *arg2);
+static int readbuffer_new(BIO *h);
+static int readbuffer_free(BIO *data);
+static long readbuffer_callback_ctrl(BIO *h, int cmd, BIO_info_cb *fp);
+
+static const BIO_METHOD methods_readbuffer = {
+    BIO_TYPE_BUFFER,
+    "readbuffer",
+    bwrite_conv,
+    readbuffer_write,
+    bread_conv,
+    readbuffer_read,
+    readbuffer_puts,
+    readbuffer_gets,
+    readbuffer_ctrl,
+    readbuffer_new,
+    readbuffer_free,
+    readbuffer_callback_ctrl,
+};
+
+const BIO_METHOD *BIO_f_readbuffer(void)
+{
+    return &methods_readbuffer;
+}
+
+static int readbuffer_new(BIO *bi)
+{
+    BIO_F_BUFFER_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));
+
+    if (ctx == NULL)
+        return 0;
+    ctx->ibuf_size = DEFAULT_BUFFER_SIZE;
+    ctx->ibuf = OPENSSL_malloc(DEFAULT_BUFFER_SIZE);
+    if (ctx->ibuf == NULL) {
+        OPENSSL_free(ctx);
+        return 0;
+    }
+
+    bi->init = 1;
+    bi->ptr = (char *)ctx;
+    bi->flags = 0;
+    return 1;
+}
+
+static int readbuffer_free(BIO *a)
+{
+    BIO_F_BUFFER_CTX *b;
+
+    if (a == NULL)
+        return 0;
+    b = (BIO_F_BUFFER_CTX *)a->ptr;
+    OPENSSL_free(b->ibuf);
+    OPENSSL_free(a->ptr);
+    a->ptr = NULL;
+    a->init = 0;
+    a->flags = 0;
+    return 1;
+}
+
+static int readbuffer_resize(BIO_F_BUFFER_CTX *ctx, int sz)
+{
+    char *tmp;
+
+    /* Figure out how many blocks are required */
+    sz += (ctx->ibuf_off + DEFAULT_BUFFER_SIZE - 1);
+    sz = DEFAULT_BUFFER_SIZE * (sz / DEFAULT_BUFFER_SIZE);
+
+    /* Resize if the buffer is not big enough */
+    if (sz > ctx->ibuf_size) {
+        tmp = OPENSSL_realloc(ctx->ibuf, sz);
+        if (tmp == NULL)
+            return 0;
+        ctx->ibuf = tmp;
+        ctx->ibuf_size = sz;
+    }
+    return 1;
+}
+
+static int readbuffer_read(BIO *b, char *out, int outl)
+{
+    int i, num = 0;
+    BIO_F_BUFFER_CTX *ctx;
+
+    if (out == NULL || outl == 0)
+        return 0;
+    ctx = (BIO_F_BUFFER_CTX *)b->ptr;
+
+    if ((ctx == NULL) || (b->next_bio == NULL))
+        return 0;
+    BIO_clear_retry_flags(b);
+
+    for (;;) {
+        i = ctx->ibuf_len;
+        /* If there is something in the buffer just read it. */
+        if (i != 0) {
+            if (i > outl)
+                i = outl;
+            memcpy(out, &(ctx->ibuf[ctx->ibuf_off]), i);
+            ctx->ibuf_off += i;
+            ctx->ibuf_len -= i;
+            num += i;
+            /* Exit if we have read the bytes required out of the buffer */
+            if (outl == i)
+                return num;
+            outl -= i;
+            out += i;
+        }
+
+        /* Only gets here if the buffer has been consumed */
+        if (!readbuffer_resize(ctx, outl))
+            return 0;
+
+        /* Do some buffering by reading from the next bio */
+        i = BIO_read(b->next_bio, ctx->ibuf + ctx->ibuf_off, outl);
+        if (i <= 0) {
+            BIO_copy_next_retry(b);
+            if (i < 0)
+                return ((num > 0) ? num : i);
+            else
+                return num; /* i == 0 */
+        }
+        ctx->ibuf_len = i;
+    }
+}
+
+static int readbuffer_write(BIO *b, const char *in, int inl)
+{
+    return 0;
+}
+static int readbuffer_puts(BIO *b, const char *str)
+{
+    return 0;
+}
+
+static long readbuffer_ctrl(BIO *b, int cmd, long num, void *ptr)
+{
+    BIO_F_BUFFER_CTX *ctx;
+    long ret = 1, sz;
+
+    ctx = (BIO_F_BUFFER_CTX *)b->ptr;
+
+    switch (cmd) {
+    case BIO_CTRL_EOF:
+        if (ctx->ibuf_len > 0)
+            return 0;
+        if (b->next_bio == NULL)
+            return 1;
+        ret = BIO_ctrl(b->next_bio, cmd, num, ptr);
+        break;
+
+    case BIO_C_FILE_SEEK:
+    case BIO_CTRL_RESET:
+        sz = ctx->ibuf_off + ctx->ibuf_len;
+        /* Assume it can only seek backwards */
+        if (num < 0 || num > sz)
+            return 0;
+        ctx->ibuf_off = num;
+        ctx->ibuf_len = sz - num;
+        break;
+
+    case BIO_C_FILE_TELL:
+    case BIO_CTRL_INFO:
+        ret = (long)ctx->ibuf_off;
+        break;
+    case BIO_CTRL_PENDING:
+        ret = (long)ctx->ibuf_len;
+        if (ret == 0) {
+            if (b->next_bio == NULL)
+                return 0;
+            ret = BIO_ctrl(b->next_bio, cmd, num, ptr);
+        }
+        break;
+    case BIO_CTRL_DUP:
+    case BIO_CTRL_FLUSH:
+        ret = 1;
+        break;
+    default:
+        ret = 0;
+        break;
+    }
+    return ret;
+}
+
+static long readbuffer_callback_ctrl(BIO *b, int cmd, BIO_info_cb *fp)
+{
+    if (b->next_bio == NULL)
+        return 0;
+    return BIO_callback_ctrl(b->next_bio, cmd, fp);
+}
+
+static int readbuffer_gets(BIO *b, char *buf, int size)
+{
+    BIO_F_BUFFER_CTX *ctx;
+    int num = 0, num_chars, found_newline;
+    char *p;
+
+    if (size == 0)
+        return 0;
+    --size; /* the passed in size includes the terminator - so remove it here */
+    ctx = (BIO_F_BUFFER_CTX *)b->ptr;
+    BIO_clear_retry_flags(b);
+
+    for (;;) {
+        if (ctx->ibuf_len > 0) {
+            p = &(ctx->ibuf[ctx->ibuf_off]);
+            found_newline = 0;
+            for (num_chars = 0;
+                 (num_chars < ctx->ibuf_len) && (num_chars < size);
+                 num_chars++) {
+                *(buf++) = p[num_chars];
+                if (p[num_chars] == '\n') {
+                    found_newline = 1;
+                    num_chars++;
+                    break;
+                }
+            }
+            num += num_chars;
+            size -= num_chars;
+            ctx->ibuf_len -= num_chars;
+            ctx->ibuf_off += num_chars;
+            if (found_newline || size == 0) {
+                *buf = '\0';
+                return num;
+            }
+        } else {
+            /* read another line and resize if we have to */
+            if (!readbuffer_resize(ctx, size))
+                return 0;
+
+            /* Read another line from the next bio using BIO_gets */
+            num_chars = BIO_gets(b->next_bio, ctx->ibuf + ctx->ibuf_off,
+                                 1 + size);
+            if (num_chars <= 0) {
+                BIO_copy_next_retry(b);
+                *buf = '\0';
+                return num > 0 ? num : num_chars;
+            }
+            ctx->ibuf_len = num_chars;
+        }
+    }
+}

crypto/bio/build.info

@@ -15,4 +15,4 @@ SOURCE[../../libcrypto]=\
 
 # Filters
 SOURCE[../../libcrypto]=\
-        bf_null.c bf_buff.c bf_lbuf.c bf_nbio.c bf_prefix.c
+        bf_null.c bf_buff.c bf_lbuf.c bf_nbio.c bf_prefix.c bf_readbuff.c

doc/build.info

@@ -570,6 +570,10 @@ DEPEND[html/man3/BIO_f_prefix.html]=man3/BIO_f_prefix.pod
 GENERATE[html/man3/BIO_f_prefix.html]=man3/BIO_f_prefix.pod
 DEPEND[man/man3/BIO_f_prefix.3]=man3/BIO_f_prefix.pod
 GENERATE[man/man3/BIO_f_prefix.3]=man3/BIO_f_prefix.pod
+DEPEND[html/man3/BIO_f_readbuffer.html]=man3/BIO_f_readbuffer.pod
+GENERATE[html/man3/BIO_f_readbuffer.html]=man3/BIO_f_readbuffer.pod
+DEPEND[man/man3/BIO_f_readbuffer.3]=man3/BIO_f_readbuffer.pod
+GENERATE[man/man3/BIO_f_readbuffer.3]=man3/BIO_f_readbuffer.pod
 DEPEND[html/man3/BIO_f_ssl.html]=man3/BIO_f_ssl.pod
 GENERATE[html/man3/BIO_f_ssl.html]=man3/BIO_f_ssl.pod
 DEPEND[man/man3/BIO_f_ssl.3]=man3/BIO_f_ssl.pod
@@ -2772,6 +2776,7 @@ html/man3/BIO_f_cipher.html \
 html/man3/BIO_f_md.html \
 html/man3/BIO_f_null.html \
 html/man3/BIO_f_prefix.html \
+html/man3/BIO_f_readbuffer.html \
 html/man3/BIO_f_ssl.html \
 html/man3/BIO_find_type.html \
 html/man3/BIO_get_data.html \
@@ -3342,6 +3347,7 @@ man/man3/BIO_f_cipher.3 \
 man/man3/BIO_f_md.3 \
 man/man3/BIO_f_null.3 \
 man/man3/BIO_f_prefix.3 \
+man/man3/BIO_f_readbuffer.3 \
 man/man3/BIO_f_ssl.3 \
 man/man3/BIO_find_type.3 \
 man/man3/BIO_get_data.3 \

doc/man3/BIO_f_readbuffer.pod

@@ -0,0 +1,61 @@
+=pod
+
+=head1 NAME
+
+BIO_f_readbuffer
+- read only buffering BIO that supports BIO_tell() and BIO_seek()
+
+=head1 SYNOPSIS
+
+ #include <openssl/bio.h>
+
+ const BIO_METHOD *BIO_f_readbuffer(void);
+
+=head1 DESCRIPTION
+
+BIO_f_readbuffer() returns the read buffering BIO method.
+
+This BIO filter can be inserted on top of BIO's that do not support BIO_tell()
+or BIO_seek() (e.g. A file BIO that uses stdin).
+
+Data read from a read buffering BIO comes from an internal buffer which is
+filled from the next BIO in the chain.
+
+BIO_gets() is supported for read buffering BIOs.
+Writing data to a read buffering BIO is not supported.
+
+Calling BIO_reset() on a read buffering BIO does not clear any buffered data.
+
+=head1 NOTES
+
+Read buffering BIOs implement BIO_read_ex() by using BIO_read_ex() operations
+on the next BIO (e.g. a file BIO) in the chain and storing the result in an
+internal buffer, from which bytes are given back to the caller as appropriate
+for the call. BIO_read_ex() is guaranteed to give the caller the number of bytes
+it asks for, unless there's an error or end of communication is reached in the
+next BIO. The internal buffer can grow to cache the entire contents of the next
+BIO in the chain. BIO_seek() uses the internal buffer, so that it can only seek
+into data that is already read.
+
+=head1 RETURN VALUES
+
+BIO_f_readbuffer() returns the read buffering BIO method.
+
+=head1 SEE ALSO
+
+L<bio(7)>,
+L<BIO_read(3)>,
+L<BIO_gets(3)>,
+L<BIO_reset(3)>,
+L<BIO_ctrl(3)>.
+
+=head1 COPYRIGHT
+
+Copyright 2021 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut

doc/man7/bio.pod

@@ -64,6 +64,7 @@ L<BIO_ctrl(3)>,
 L<BIO_f_base64(3)>, L<BIO_f_buffer(3)>,
 L<BIO_f_cipher(3)>, L<BIO_f_md(3)>,
 L<BIO_f_null(3)>, L<BIO_f_ssl(3)>,
+L<BIO_f_readbuffer(3)>,
 L<BIO_find_type(3)>, L<BIO_new(3)>,
 L<BIO_new_bio_pair(3)>,
 L<BIO_push(3)>, L<BIO_read_ex(3)>,
@@ -76,7 +77,7 @@ L<BIO_should_retry(3)>
 
 =head1 COPYRIGHT
 
-Copyright 2000-2017 The OpenSSL Project Authors. All Rights Reserved.
+Copyright 2000-2021 The OpenSSL Project Authors. All Rights Reserved.
 
 Licensed under the Apache License 2.0 (the "License").  You may not use
 this file except in compliance with the License.  You can obtain a copy

include/openssl/bio.h.in

@@ -647,6 +647,7 @@ const BIO_METHOD *BIO_s_bio(void);
 const BIO_METHOD *BIO_s_null(void);
 const BIO_METHOD *BIO_f_null(void);
 const BIO_METHOD *BIO_f_buffer(void);
+const BIO_METHOD *BIO_f_readbuffer(void);
 const BIO_METHOD *BIO_f_linebuffer(void);
 const BIO_METHOD *BIO_f_nbio_test(void);
 const BIO_METHOD *BIO_f_prefix(void);