Adapt other parts of the source to the changed EVP_Q_digest() and EVP_Q_mac()

Fixes #15839

Reviewed-by: David von Oheimb <david.von.oheimb@siemens.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15861)

apps/lib/s_cb.c

@@ -740,8 +740,8 @@ void tlsext_cb(SSL *s, int client_server, int type,
 }
 
 #ifndef OPENSSL_NO_SOCK
-int generate_cookie_callback(SSL *ssl, unsigned char *cookie,
-                             unsigned int *cookie_len)
+int generate_stateless_cookie_callback(SSL *ssl, unsigned char *cookie,
+                                       size_t *cookie_len)
 {
     unsigned char *buffer = NULL;
     size_t length = 0;
@@ -800,16 +800,16 @@ end:
     return res;
 }
 
-int verify_cookie_callback(SSL *ssl, const unsigned char *cookie,
-                           unsigned int cookie_len)
+int verify_stateless_cookie_callback(SSL *ssl, const unsigned char *cookie,
+                                     size_t cookie_len)
 {
     unsigned char result[EVP_MAX_MD_SIZE];
-    unsigned int resultlength;
+    size_t resultlength;
 
     /* Note: we check cookie_initialized because if it's not,
      * it cannot be valid */
     if (cookie_initialized
-        && generate_cookie_callback(ssl, result, &resultlength)
+        && generate_stateless_cookie_callback(ssl, result, &resultlength)
         && cookie_len == resultlength
         && memcmp(result, cookie, resultlength) == 0)
         return 1;
@@ -817,20 +817,20 @@ int verify_cookie_callback(SSL *ssl, const unsigned char *cookie,
     return 0;
 }
 
-int generate_stateless_cookie_callback(SSL *ssl, unsigned char *cookie,
-                                       size_t *cookie_len)
+int generate_cookie_callback(SSL *ssl, unsigned char *cookie,
+                             unsigned int *cookie_len)
 {
-    unsigned int temp = 0;
+    size_t temp = 0;
+    int res = generate_stateless_cookie_callback(ssl, cookie, &temp);
 
-    int res = generate_cookie_callback(ssl, cookie, &temp);
-    *cookie_len = temp;
+    *cookie_len = (unsigned int)temp;
     return res;
 }
 
-int verify_stateless_cookie_callback(SSL *ssl, const unsigned char *cookie,
-                                     size_t cookie_len)
+int verify_cookie_callback(SSL *ssl, const unsigned char *cookie,
+                           unsigned int cookie_len)
 {
-    return verify_cookie_callback(ssl, cookie, cookie_len);
+    return verify_stateless_cookie_callback(ssl, cookie, cookie_len);
 }
 
 #endif

crypto/crmf/crmf_pbm.c

@@ -140,7 +140,6 @@ int OSSL_CRMF_pbm_new(OSSL_LIB_CTX *libctx, const char *propq,
     unsigned int bklen = EVP_MAX_MD_SIZE;
     int64_t iterations;
     unsigned char *mac_res = 0;
-    unsigned int maclen;
     int ok = 0;
 
     if (out == NULL || pbmp == NULL || pbmp->mac == NULL
@@ -207,10 +206,9 @@ int OSSL_CRMF_pbm_new(OSSL_LIB_CTX *libctx, const char *propq,
         goto err;
     }
     if (EVP_Q_mac(libctx, "HMAC", propq, hmac_mdname, NULL, basekey, bklen,
-                  msg, msglen, mac_res, EVP_MAX_MD_SIZE, &maclen) == NULL)
+                  msg, msglen, mac_res, EVP_MAX_MD_SIZE, outlen) == NULL)
         goto err;
 
-    *outlen = (size_t)maclen;
     ok = 1;
 
  err:

crypto/hmac/hmac.c

@@ -224,12 +224,17 @@ unsigned char *HMAC(const EVP_MD *evp_md, const void *key, int key_len,
 {
     static unsigned char static_md[EVP_MAX_MD_SIZE];
     int size = EVP_MD_get_size(evp_md);
+    size_t temp_md_len = 0;
+    unsigned char *ret = NULL;
 
-    if (size < 0)
-        return NULL;
-    return EVP_Q_mac(NULL, "HMAC", NULL, EVP_MD_get0_name(evp_md), NULL,
-                     key, key_len, data, data_len,
-                     md == NULL ? static_md : md, size, md_len);
+    if (size >= 0) {
+        ret = EVP_Q_mac(NULL, "HMAC", NULL, EVP_MD_get0_name(evp_md), NULL,
+                        key, key_len, data, data_len,
+                        md == NULL ? static_md : md, size, &temp_md_len);
+        if (md_len != NULL)
+            *md_len = (unsigned int)temp_md_len;
+    }
+    return ret;
 }
 
 void HMAC_CTX_set_flags(HMAC_CTX *ctx, unsigned long flags)

ssl/tls13_enc.c

@@ -309,8 +309,7 @@ size_t tls13_final_finish_mac(SSL *s, const char *str, size_t slen,
     unsigned char hash[EVP_MAX_MD_SIZE];
     unsigned char finsecret[EVP_MAX_MD_SIZE];
     unsigned char *key = NULL;
-    unsigned int len = 0;
-    size_t hashlen, ret = 0;
+    size_t len = 0, hashlen;
     OSSL_PARAM params[2], *p = params;
 
     /* Safe to cast away const here since we're not "getting" any data */
@@ -345,10 +344,9 @@ size_t tls13_final_finish_mac(SSL *s, const char *str, size_t slen,
         goto err;
     }
 
-    ret = len;
  err:
     OPENSSL_cleanse(finsecret, sizeof(finsecret));
-    return ret;
+    return len;
 }
 
 /*