Don't allow SHAKE128/SHAKE256 with HMAC

See discussion in github issue #8563

Fixes #8563

Reviewed-by: Matthias St. Pierre <Matthias.St.Pierre@ncp-e.com>
(Merged from https://github.com/openssl/openssl/pull/8584)

crypto/hmac/hmac.c

@@ -35,6 +35,13 @@ int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len,
         return 0;
     }
 
+    /*
+     * The HMAC construction is not allowed to be used with the
+     * extendable-output functions (XOF) shake128 and shake256.
+     */
+    if ((EVP_MD_meth_get_flags(md) & EVP_MD_FLAG_XOF) != 0)
+        return 0;
+
     if (key != NULL) {
         reset = 1;
         j = EVP_MD_block_size(md);

doc/man3/EVP_MAC.pod

@@ -202,6 +202,9 @@ For MAC implementations that use an underlying computation algorithm,
 these controls set what the algorithm should be, and the engine that
 implements the algorithm if needed.
 
+Note that not all algorithms may support all digests. HMAC does not support
+variable output length digests such as SHAKE128 or SHAKE256.
+
 B<EVP_MAC_CTRL_SET_ENGINE> takes one argument: C<ENGINE *>
 
 B<EVP_MAC_CTRL_SET_MD> takes one argument: C<EVP_MD *>

doc/man3/HMAC.pod

@@ -63,7 +63,9 @@ If B<md> is NULL, the digest is placed in a static array.  The size of
 the output is placed in B<md_len>, unless it is B<NULL>. Note: passing a NULL
 value for B<md>  to use the static array is not thread safe.
 
-B<evp_md> can be EVP_sha1(), EVP_ripemd160() etc.
+B<evp_md> is a message digest such as EVP_sha1(), EVP_ripemd160() etc. HMAC does
+not support variable output length digests such as EVP_shake128() and
+EVP_shake256().
 
 HMAC_CTX_new() creates a new HMAC_CTX in heap memory.
 

test/recipes/30-test_evp_data/evpmac.txt

@@ -589,6 +589,14 @@ Input = "Sample message for keylen>blocklen"
 Key = 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f8081828384858687
 Output = 5f464f5e5b7848e3885e49b2c385f0694985d0e38966242dc4a5fe3fea4b37d46b65ceced5dcf59438dd840bab22269f0ba7febdb9fcf74602a35666b2a32915
 
+Title = HMAC self generated tests
+
+MAC = HMAC
+Algorithm = SHAKE128
+Input = "Test that SHAKE128 fails"
+Key = 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
+Result = MAC_CTRL_ERROR
+
 
 Title = CMAC tests (from FIPS module)