Add SHA256 and SHA512 based output for 'openssl passwd'

RT#4674 issue 2

Reviewed-by: Rich Salz <rsalz@openssl.org>

apps/passwd.c

@@ -11,7 +11,11 @@
 # define NO_MD5CRYPT_1
 #endif
 
-#if !defined(OPENSSL_NO_DES) || !defined(NO_MD5CRYPT_1)
+#if defined OPENSSL_NO_SHA || defined CHARSET_EBCDIC
+# define NO_SHACRYPT
+#endif
+
+#if !defined(OPENSSL_NO_DES) || !defined(NO_MD5CRYPT_1) || !defined(NO_SHACRYPT)
 
 # include <string.h>
 
@@ -27,6 +31,9 @@
 # ifndef NO_MD5CRYPT_1
 #  include <openssl/md5.h>
 # endif
+# ifndef NO_SHACRYPT
+#  include <openssl/sha.h>
+# endif
 
 static unsigned const char cov_2char[64] = {
     /* from crypto/des/fcrypt.c */
@@ -43,13 +50,13 @@ static unsigned const char cov_2char[64] = {
 static int do_passwd(int passed_salt, char **salt_p, char **salt_malloc_p,
                      char *passwd, BIO *out, int quiet, int table,
                      int reverse, size_t pw_maxlen, int usecrypt, int use1,
-                     int useapr1);
+                     int useapr1, int use5, int use6);
 
 typedef enum OPTION_choice {
     OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
     OPT_IN,
     OPT_NOVERIFY, OPT_QUIET, OPT_TABLE, OPT_REVERSE, OPT_APR1,
-    OPT_1, OPT_CRYPT, OPT_SALT, OPT_STDIN
+    OPT_1, OPT_5, OPT_6, OPT_CRYPT, OPT_SALT, OPT_STDIN
 } OPTION_CHOICE;
 
 OPTIONS passwd_options[] = {
@@ -62,6 +69,10 @@ OPTIONS passwd_options[] = {
     {"reverse", OPT_REVERSE, '-', "Switch table columns"},
     {"salt", OPT_SALT, 's', "Use provided salt"},
     {"stdin", OPT_STDIN, '-', "Read passwords from stdin"},
+# ifndef NO_SHACRYPT
+    {"6", OPT_6, '-', "SHA512-based password algorithm"},
+    {"5", OPT_5, '-', "SHA256-based password algorithm"},
+# endif
 # ifndef NO_MD5CRYPT_1
     {"apr1", OPT_APR1, '-', "MD5-based password algorithm, Apache variant"},
     {"1", OPT_1, '-', "MD5-based password algorithm"},
@@ -83,7 +94,7 @@ int passwd_main(int argc, char **argv)
     int in_noverify = 0;
 #endif
     int passed_salt = 0, quiet = 0, table = 0, reverse = 0;
-    int ret = 1, usecrypt = 0, use1 = 0, useapr1 = 0;
+    int ret = 1, usecrypt = 0, use1 = 0, useapr1 = 0, use5 = 0, use6 = 0;
     size_t passwd_malloc_size = 0, pw_maxlen = 256;
 
     prog = opt_init(argc, argv, passwd_options);
@@ -118,6 +129,12 @@ int passwd_main(int argc, char **argv)
         case OPT_REVERSE:
             reverse = 1;
             break;
+        case OPT_5:
+            use5 = 1;
+            break;
+        case OPT_6:
+            use6 = 1;
+            break;
         case OPT_1:
             use1 = 1;
             break;
@@ -149,11 +166,11 @@ int passwd_main(int argc, char **argv)
         passwds = argv;
     }
 
-    if (!usecrypt && !use1 && !useapr1) {
+    if (!usecrypt && !use5 && !use6 && !use1 && !useapr1) {
         /* use default */
         usecrypt = 1;
     }
-    if (usecrypt + use1 + useapr1 > 1) {
+    if (usecrypt + use5 + use6 + use1 + useapr1 > 1) {
         /* conflict */
         goto opthelp;
     }
@@ -166,6 +183,10 @@ int passwd_main(int argc, char **argv)
     if (use1 || useapr1)
         goto opthelp;
 # endif
+# ifdef NO_SHACRYPT
+    if (use5 || use6)
+        goto opthelp;
+# endif
 
     if (infile != NULL && in_stdin) {
         BIO_printf(bio_err, "%s: Can't combine -in and -stdin\n", prog);
@@ -226,7 +247,7 @@ int passwd_main(int argc, char **argv)
             passwd = *passwds++;
             if (!do_passwd(passed_salt, &salt, &salt_malloc, passwd, bio_out,
                            quiet, table, reverse, pw_maxlen, usecrypt, use1,
-                           useapr1))
+                           useapr1, use5, use6))
                 goto end;
         }
         while (*passwds != NULL);
@@ -252,7 +273,8 @@ int passwd_main(int argc, char **argv)
 
                 if (!do_passwd
                     (passed_salt, &salt, &salt_malloc, passwd, bio_out, quiet,
-                     table, reverse, pw_maxlen, usecrypt, use1, useapr1))
+                     table, reverse, pw_maxlen, usecrypt, use1, useapr1,
+                     use5, use6))
                     goto end;
             }
             done = (r <= 0);
@@ -419,10 +441,270 @@ static char *md5crypt(const char *passwd, const char *magic, const char *salt)
 }
 # endif
 
+# ifndef NO_SHACRYPT
+/*
+ * SHA based password algorithm, describe by Ulrich Drepper here:
+ * https://www.akkadia.org/drepper/SHA-crypt.txt
+ * (note that it's in the public domain)
+ */
+static char *shacrypt(const char *passwd, const char *magic, const char *salt)
+{
+    /* Prefix for optional rounds specification.  */
+    static const char rounds_prefix[] = "rounds=";
+    /* Maximum salt string length.  */
+#define SALT_LEN_MAX 16
+    /* Default number of rounds if not explicitly specified.  */
+#define ROUNDS_DEFAULT 5000
+    /* Minimum number of rounds.  */
+#define ROUNDS_MIN 1000
+    /* Maximum number of rounds.  */
+#define ROUNDS_MAX 999999999
+
+    /* "$6$rounds=<N>$......salt......$...shahash(up to 86 chars)...\0" */
+    static char out_buf[3 + 17 + 17 + 86 + 1];
+    unsigned char buf[SHA512_DIGEST_LENGTH];
+    unsigned char temp_buf[SHA512_DIGEST_LENGTH];
+    size_t buf_size = 0;
+    char salt_copy[17];          /* Max 16 chars plus '\0' */
+    size_t n;
+    EVP_MD_CTX *md = NULL, *md2 = NULL;
+    const EVP_MD *sha = NULL;
+    size_t passwd_len, salt_len, magic_len;
+    size_t rounds = 5000;        /* Default */
+    char rounds_custom = 0;
+    char *p_bytes = NULL;
+    char *s_bytes = NULL;
+    char *cp = NULL;
+
+    passwd_len = strlen(passwd);
+    magic_len = strlen(magic);
+
+    /* assert it's "5" or "6" */
+    if (magic_len != 1)
+        return NULL;
+
+    switch (magic[0]) {
+    case '5':
+        sha = EVP_sha256();
+        buf_size = 32;
+        break;
+    case '6':
+        sha = EVP_sha512();
+        buf_size = 64;
+        break;
+    default:
+        return NULL;
+    }
+
+    if (strncmp(salt, rounds_prefix, sizeof(rounds_prefix) - 1) == 0) {
+        const char *num = salt + sizeof(rounds_prefix) - 1;
+        char *endp;
+        unsigned long int srounds = strtoul (num, &endp, 10);
+        if (*endp == '$') {
+            salt = endp + 1;
+            if (srounds > ROUNDS_MAX)
+                rounds = ROUNDS_MAX;
+            else if (srounds < ROUNDS_MIN)
+                rounds = ROUNDS_MIN;
+            else
+                rounds = srounds;
+            rounds_custom = 1;
+        } else {
+            return NULL;
+        }
+    }
+
+    /* The salt gets truncated to 16 chars */
+    OPENSSL_strlcpy(salt_copy, salt, sizeof salt_copy);
+    salt_len = strlen(salt_copy);
+
+    out_buf[0] = 0;
+    OPENSSL_strlcat(out_buf, "$", sizeof out_buf);
+    OPENSSL_strlcat(out_buf, magic, sizeof out_buf);
+    OPENSSL_strlcat(out_buf, "$", sizeof out_buf);
+    if (rounds_custom) {
+        char tmp_buf[7 + 9 + 1]; /* "rounds=999999999" */
+        sprintf(tmp_buf, "rounds=%lu", rounds);
+        OPENSSL_strlcat(out_buf, tmp_buf, sizeof out_buf);
+        OPENSSL_strlcat(out_buf, "$", sizeof out_buf);
+    }
+    OPENSSL_strlcat(out_buf, salt_copy, sizeof out_buf);
+
+    /* assert "$5$rounds=999999999$......salt......" */
+    if (strlen(out_buf) > 3 + 17 * rounds_custom + salt_len )
+        return NULL;
+
+    md = EVP_MD_CTX_new();
+    if (md == NULL
+        || !EVP_DigestInit_ex(md, sha, NULL)
+        || !EVP_DigestUpdate(md, passwd, passwd_len)
+        || !EVP_DigestUpdate(md, salt_copy, salt_len))
+        goto err;
+
+    md2 = EVP_MD_CTX_new();
+    if (md2 == NULL
+        || !EVP_DigestInit_ex(md2, sha, NULL)
+        || !EVP_DigestUpdate(md2, passwd, passwd_len)
+        || !EVP_DigestUpdate(md2, salt_copy, salt_len)
+        || !EVP_DigestUpdate(md2, passwd, passwd_len)
+        || !EVP_DigestFinal_ex(md2, buf, NULL))
+        goto err;
+
+    for (n = passwd_len; n > buf_size; n -= buf_size) {
+        if (!EVP_DigestUpdate(md, buf, buf_size))
+            goto err;
+    }
+    if (!EVP_DigestUpdate(md, buf, n))
+        goto err;
+
+    n = passwd_len;
+    while (n) {
+        if (!EVP_DigestUpdate(md,
+                              (n & 1) ? buf : (unsigned const char *)passwd,
+                              (n & 1) ? buf_size : passwd_len))
+            goto err;
+        n >>= 1;
+    }
+    if (!EVP_DigestFinal_ex(md, buf, NULL))
+        return NULL;
+
+    /* P sequence */
+    if (!EVP_DigestInit_ex(md2, sha, NULL))
+        goto err;
+
+    for (n = passwd_len; n > 0; n--)
+        if (!EVP_DigestUpdate(md2, passwd, passwd_len))
+            goto err;
+
+    if (!EVP_DigestFinal_ex(md2, temp_buf, NULL))
+        return NULL;
+
+    if ((p_bytes = OPENSSL_zalloc(passwd_len)) == NULL)
+        goto err;
+    for (cp = p_bytes, n = passwd_len; n > buf_size; n -= buf_size, cp += buf_size)
+        memcpy(cp, temp_buf, buf_size);
+    memcpy(cp, temp_buf, n);
+
+    /* S sequence */
+    if (!EVP_DigestInit_ex(md2, sha, NULL))
+        goto err;
+
+    for (n = 16 + buf[0]; n > 0; n--)
+        if (!EVP_DigestUpdate(md2, salt, salt_len))
+            goto err;
+
+    if (!EVP_DigestFinal_ex(md2, temp_buf, NULL))
+        return NULL;
+
+    if ((s_bytes = OPENSSL_zalloc(salt_len)) == NULL)
+        goto err;
+    for (cp = s_bytes, n = salt_len; n > buf_size; n -= buf_size, cp += buf_size)
+        memcpy(cp, temp_buf, buf_size);
+    memcpy(cp, temp_buf, n);
+
+    for (n = 0; n < rounds; n++) {
+        if (!EVP_DigestInit_ex(md2, sha, NULL))
+            goto err;
+        if (!EVP_DigestUpdate(md2,
+                              (n & 1) ? (unsigned const char *)p_bytes : buf,
+                              (n & 1) ? passwd_len : buf_size))
+            goto err;
+        if (n % 3) {
+            if (!EVP_DigestUpdate(md2, s_bytes, salt_len))
+                goto err;
+        }
+        if (n % 7) {
+            if (!EVP_DigestUpdate(md2, p_bytes, passwd_len))
+                goto err;
+        }
+        if (!EVP_DigestUpdate(md2,
+                              (n & 1) ? buf : (unsigned const char *)p_bytes,
+                              (n & 1) ? buf_size : passwd_len))
+                goto err;
+        if (!EVP_DigestFinal_ex(md2, buf, NULL))
+                goto err;
+    }
+    EVP_MD_CTX_free(md2);
+    EVP_MD_CTX_free(md);
+    md2 = NULL;
+    md = NULL;
+    OPENSSL_free(p_bytes);
+    OPENSSL_free(s_bytes);
+    p_bytes = NULL;
+    s_bytes = NULL;
+
+    cp = out_buf + strlen(out_buf);
+    *cp++ = '$';
+
+#define b64_from_24bit(B2, B1, B0, N)                                   \
+    do {                                                                \
+        unsigned int w = ((B2) << 16) | ((B1) << 8) | (B0);             \
+        int i = (N);                                                    \
+        while (i-- > 0)                                                 \
+            {                                                           \
+                *cp++ = cov_2char[w & 0x3f];                            \
+                w >>= 6;                                                \
+            }                                                           \
+    } while (0)
+
+    switch (*magic) {
+    case '5':
+        b64_from_24bit (buf[0], buf[10], buf[20], 4);
+        b64_from_24bit (buf[21], buf[1], buf[11], 4);
+        b64_from_24bit (buf[12], buf[22], buf[2], 4);
+        b64_from_24bit (buf[3], buf[13], buf[23], 4);
+        b64_from_24bit (buf[24], buf[4], buf[14], 4);
+        b64_from_24bit (buf[15], buf[25], buf[5], 4);
+        b64_from_24bit (buf[6], buf[16], buf[26], 4);
+        b64_from_24bit (buf[27], buf[7], buf[17], 4);
+        b64_from_24bit (buf[18], buf[28], buf[8], 4);
+        b64_from_24bit (buf[9], buf[19], buf[29], 4);
+        b64_from_24bit (0, buf[31], buf[30], 3);
+        break;
+    case '6':
+        b64_from_24bit (buf[0], buf[21], buf[42], 4);
+        b64_from_24bit (buf[22], buf[43], buf[1], 4);
+        b64_from_24bit (buf[44], buf[2], buf[23], 4);
+        b64_from_24bit (buf[3], buf[24], buf[45], 4);
+        b64_from_24bit (buf[25], buf[46], buf[4], 4);
+        b64_from_24bit (buf[47], buf[5], buf[26], 4);
+        b64_from_24bit (buf[6], buf[27], buf[48], 4);
+        b64_from_24bit (buf[28], buf[49], buf[7], 4);
+        b64_from_24bit (buf[50], buf[8], buf[29], 4);
+        b64_from_24bit (buf[9], buf[30], buf[51], 4);
+        b64_from_24bit (buf[31], buf[52], buf[10], 4);
+        b64_from_24bit (buf[53], buf[11], buf[32], 4);
+        b64_from_24bit (buf[12], buf[33], buf[54], 4);
+        b64_from_24bit (buf[34], buf[55], buf[13], 4);
+        b64_from_24bit (buf[56], buf[14], buf[35], 4);
+        b64_from_24bit (buf[15], buf[36], buf[57], 4);
+        b64_from_24bit (buf[37], buf[58], buf[16], 4);
+        b64_from_24bit (buf[59], buf[17], buf[38], 4);
+        b64_from_24bit (buf[18], buf[39], buf[60], 4);
+        b64_from_24bit (buf[40], buf[61], buf[19], 4);
+        b64_from_24bit (buf[62], buf[20], buf[41], 4);
+        b64_from_24bit (0, 0, buf[63], 2);
+        break;
+    default:
+        goto err;
+    }
+    *cp = '\0';
+
+    return out_buf;
+
+ err:
+    EVP_MD_CTX_free(md2);
+    EVP_MD_CTX_free(md);
+    OPENSSL_free(p_bytes);
+    OPENSSL_free(s_bytes);
+    return NULL;
+}
+# endif
+
 static int do_passwd(int passed_salt, char **salt_p, char **salt_malloc_p,
                      char *passwd, BIO *out, int quiet, int table,
                      int reverse, size_t pw_maxlen, int usecrypt, int use1,
-                     int useapr1)
+                     int useapr1, int use5, int use6)
 {
     char *hash = NULL;
 
@@ -463,6 +745,22 @@ static int do_passwd(int passed_salt, char **salt_p, char **salt_malloc_p,
             (*salt_p)[8] = 0;
         }
 # endif                         /* !NO_MD5CRYPT_1 */
+
+# ifndef NO_SHACRYPT
+        if (use5 || use6) {
+            int i;
+
+            if (*salt_malloc_p == NULL) {
+                *salt_p = *salt_malloc_p = app_malloc(17, "salt buffer");
+            }
+            if (RAND_bytes((unsigned char *)*salt_p, 16) <= 0)
+                goto end;
+
+            for (i = 0; i < 16; i++)
+                (*salt_p)[i] = cov_2char[(*salt_p)[i] & 0x3f]; /* 6 bits */
+            (*salt_p)[16] = 0;
+        }
+# endif                         /* !NO_SHACRYPT */
     }
 
     assert(*salt_p != NULL);
@@ -488,6 +786,10 @@ static int do_passwd(int passed_salt, char **salt_p, char **salt_malloc_p,
 # ifndef NO_MD5CRYPT_1
     if (use1 || useapr1)
         hash = md5crypt(passwd, (use1 ? "1" : "apr1"), *salt_p);
+# endif
+# ifndef NO_SHACRYPT
+    if (use5 || use6)
+        hash = shacrypt(passwd, (use5 ? "5" : "6"), *salt_p);
 # endif
     assert(hash != NULL);