openldap/libraries/libldap/utf-8-conv.c

/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
 *
 * Copyright 1998-2020 The OpenLDAP Foundation.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
/* Portions Copyright (C) 1999, 2000 Novell, Inc. All Rights Reserved.
 * 
 * THIS WORK IS SUBJECT TO U.S. AND INTERNATIONAL COPYRIGHT LAWS AND
 * TREATIES. USE, MODIFICATION, AND REDISTRIBUTION OF THIS WORK IS SUBJECT
 * TO VERSION 2.0.1 OF THE OPENLDAP PUBLIC LICENSE, A COPY OF WHICH IS
 * AVAILABLE AT HTTP://WWW.OPENLDAP.ORG/LICENSE.HTML OR IN THE FILE "LICENSE"
 * IN THE TOP-LEVEL DIRECTORY OF THE DISTRIBUTION. ANY USE OR EXPLOITATION
 * OF THIS WORK OTHER THAN AS AUTHORIZED IN VERSION 2.0.1 OF THE OPENLDAP
 * PUBLIC LICENSE, OR OTHER PRIOR WRITTEN CONSENT FROM NOVELL, COULD SUBJECT
 * THE PERPETRATOR TO CRIMINAL AND CIVIL LIABILITY. 
 *---
 * Note: A verbatim copy of version 2.0.1 of the OpenLDAP Public License 
 * can be found in the file "build/LICENSE-2.0.1" in this distribution
 * of OpenLDAP Software.
 */

/*
 * UTF-8 Conversion Routines
 *
 * These routines convert between Wide Character and UTF-8,
 * or between MultiByte and UTF-8 encodings.
 *
 * Both single character and string versions of the functions are provided.
 * All functions return -1 if the character or string cannot be converted.
 */

#include "portable.h"

#if SIZEOF_WCHAR_T >= 4
/* These routines assume ( sizeof(wchar_t) >= 4 ) */

#include <stdio.h>
#include <ac/stdlib.h>		/* For wctomb, wcstombs, mbtowc, mbstowcs */
#include <ac/string.h>
#include <ac/time.h>		/* for time_t */

#include "ldap-int.h"

#include <ldap_utf8.h>

static unsigned char mask[] = { 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };


/*-----------------------------------------------------------------------------
					UTF-8 Format Summary

ASCII chars 						7 bits
    0xxxxxxx
    
2-character UTF-8 sequence:        11 bits
    110xxxxx  10xxxxxx

3-character UTF-8                  16 bits
    1110xxxx  10xxxxxx  10xxxxxx   
    
4-char UTF-8                       21 bits 
    11110xxx  10xxxxxx  10xxxxxx  10xxxxxx
    
5-char UTF-8                       26 bits
    111110xx  10xxxxxx  10xxxxxx  10xxxxxx  10xxxxxx
    
6-char UTF-8                       31 bits
    1111110x  10xxxxxx  10xxxxxx  10xxxxxx  10xxxxxx  10xxxxxx
    
Unicode address space   (0 - 0x10FFFF)    21 bits
ISO-10646 address space (0 - 0x7FFFFFFF)  31 bits

Note: This code does not prevent UTF-8 sequences which are longer than
      necessary from being decoded.
*/

/*----------------------------------------------------------------------------- 
   Convert a UTF-8 character to a wide char. 
   Return the length of the UTF-8 input character in bytes.
*/
int
ldap_x_utf8_to_wc ( wchar_t *wchar, const char *utf8char )
{
	int utflen, i;
	wchar_t ch;

	if (utf8char == NULL) return -1;

	/* Get UTF-8 sequence length from 1st byte */
	utflen = LDAP_UTF8_CHARLEN2(utf8char, utflen);
	
	if( utflen==0 || utflen > (int)LDAP_MAX_UTF8_LEN ) return -1;

	/* First byte minus length tag */
	ch = (wchar_t)(utf8char[0] & mask[utflen]);
	
	for(i=1; i < utflen; i++) {
		/* Subsequent bytes must start with 10 */
		if ((utf8char[i] & 0xc0) != 0x80) return -1;
	
		ch <<= 6;			/* 6 bits of data in each subsequent byte */
		ch |= (wchar_t)(utf8char[i] & 0x3f);
	}
	
	if (wchar) *wchar = ch;

	return utflen;
}

/*-----------------------------------------------------------------------------
   Convert a UTF-8 string to a wide char string.
   No more than 'count' wide chars will be written to the output buffer.
   Return the size of the converted string in wide chars, excl null terminator.
*/
int
ldap_x_utf8s_to_wcs ( wchar_t *wcstr, const char *utf8str, size_t count )
{
	size_t wclen = 0;
	int utflen, i;
	wchar_t ch;


	/* If input ptr is NULL or empty... */
	if (utf8str == NULL || !*utf8str) {
		if ( wcstr )
			*wcstr = 0;
		return 0;
	}

	/* Examine next UTF-8 character.  If output buffer is NULL, ignore count */
	while ( *utf8str && (wcstr==NULL || wclen<count) ) {
		/* Get UTF-8 sequence length from 1st byte */
		utflen = LDAP_UTF8_CHARLEN2(utf8str, utflen);
		
		if( utflen==0 || utflen > (int)LDAP_MAX_UTF8_LEN ) return -1;

		/* First byte minus length tag */
		ch = (wchar_t)(utf8str[0] & mask[utflen]);
		
		for(i=1; i < utflen; i++) {
			/* Subsequent bytes must start with 10 */
			if ((utf8str[i] & 0xc0) != 0x80) return -1;
		
			ch <<= 6;			/* 6 bits of data in each subsequent byte */
			ch |= (wchar_t)(utf8str[i] & 0x3f);
		}
		
		if (wcstr) wcstr[wclen] = ch;
		
		utf8str += utflen;	/* Move to next UTF-8 character */
		wclen++;			/* Count number of wide chars stored/required */
	}

	/* Add null terminator if there's room in the buffer. */
	if (wcstr && wclen < count) wcstr[wclen] = 0;

	return wclen;
}


/*----------------------------------------------------------------------------- 
   Convert one wide char to a UTF-8 character.
   Return the length of the converted UTF-8 character in bytes.
   No more than 'count' bytes will be written to the output buffer.
*/
int
ldap_x_wc_to_utf8 ( char *utf8char, wchar_t wchar, size_t count )
{
	int len=0;

	if (utf8char == NULL)   /* Just determine the required UTF-8 char length. */
	{						/* Ignore count */
		if( wchar < 0 )
			return -1;
		if( wchar < 0x80 )
			return 1;
		if( wchar < 0x800 )
			return 2; 
		if( wchar < 0x10000 )
			return 3;
		if( wchar < 0x200000 ) 
			return 4;
		if( wchar < 0x4000000 ) 
			return 5;
#if SIZEOF_WCHAR_T > 4
		/* UL is not strictly needed by ANSI C */
		if( wchar < (wchar_t)0x80000000UL )
#endif /* SIZEOF_WCHAR_T > 4 */
			return 6;
		return -1;
	}

	
	if ( wchar < 0 ) {				/* Invalid wide character */
		len = -1;

	} else if( wchar < 0x80 ) {
		if (count >= 1) {
			utf8char[len++] = (char)wchar;
		}

	} else if( wchar < 0x800 ) {
		if (count >=2) {
			utf8char[len++] = 0xc0 | ( wchar >> 6 );
			utf8char[len++] = 0x80 | ( wchar & 0x3f );
		}

	} else if( wchar < 0x10000 ) {
		if (count >= 3) {	
			utf8char[len++] = 0xe0 | ( wchar >> 12 );
			utf8char[len++] = 0x80 | ( (wchar >> 6) & 0x3f );
			utf8char[len++] = 0x80 | ( wchar & 0x3f );
		}
	
	} else if( wchar < 0x200000 ) {
		if (count >= 4) {
			utf8char[len++] = 0xf0 | ( wchar >> 18 );
			utf8char[len++] = 0x80 | ( (wchar >> 12) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 6) & 0x3f );
			utf8char[len++] = 0x80 | ( wchar & 0x3f );
		}

	} else if( wchar < 0x4000000 ) {
		if (count >= 5) {
			utf8char[len++] = 0xf8 | ( wchar >> 24 );
			utf8char[len++] = 0x80 | ( (wchar >> 18) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 12) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 6) & 0x3f );
			utf8char[len++] = 0x80 | ( wchar & 0x3f );
		}

	} else
#if SIZEOF_WCHAR_T > 4
		/* UL is not strictly needed by ANSI C */
		if( wchar < (wchar_t)0x80000000UL )
#endif /* SIZEOF_WCHAR_T > 4 */
	{
		if (count >= 6) {
			utf8char[len++] = 0xfc | ( wchar >> 30 );
			utf8char[len++] = 0x80 | ( (wchar >> 24) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 18) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 12) & 0x3f );
			utf8char[len++] = 0x80 | ( (wchar >> 6) & 0x3f );
			utf8char[len++] = 0x80 | ( wchar & 0x3f );
		}

#if SIZEOF_WCHAR_T > 4
	} else {
		len = -1;
#endif /* SIZEOF_WCHAR_T > 4 */
	}
	
	return len;

}


/*-----------------------------------------------------------------------------
   Convert a wide char string to a UTF-8 string.
   No more than 'count' bytes will be written to the output buffer.
   Return the # of bytes written to the output buffer, excl null terminator.
*/
int
ldap_x_wcs_to_utf8s ( char *utf8str, const wchar_t *wcstr, size_t count )
{
	int len = 0;
	int n;
	char *p = utf8str;
	wchar_t empty = 0;		/* To avoid use of L"" construct */

	if (wcstr == NULL)		/* Treat input ptr NULL as an empty string */
		wcstr = &empty;

	if (utf8str == NULL)	/* Just compute size of output, excl null */
	{
		while (*wcstr)
		{
			/* Get UTF-8 size of next wide char */
			n = ldap_x_wc_to_utf8( NULL, *wcstr++, LDAP_MAX_UTF8_LEN);
			if (n == -1)
				return -1;
			len += n;
		}

		return len;
	}

	
	/* Do the actual conversion. */

	n = 1;					/* In case of empty wcstr */
	while (*wcstr)
	{
		n = ldap_x_wc_to_utf8( p, *wcstr++, count);
		
		if (n <= 0)  		/* If encoding error (-1) or won't fit (0), quit */
			break;
		
		p += n;
		count -= n;			/* Space left in output buffer */
	}

	/* If not enough room for last character, pad remainder with null
	   so that return value = original count, indicating buffer full. */
	if (n == 0)
	{
		while (count--)
			*p++ = 0;
	}

	/* Add a null terminator if there's room. */
	else if (count)
		*p = 0;

	if (n == -1)			/* Conversion encountered invalid wide char. */
		return -1;

	/* Return the number of bytes written to output buffer, excl null. */ 
	return (p - utf8str);
}

#ifdef ANDROID
int wctomb(char *s, wchar_t wc) { return wcrtomb(s,wc,NULL); }
int mbtowc(wchar_t *pwc, const char *s, size_t n) { return mbrtowc(pwc, s, n, NULL); }
#endif

/*-----------------------------------------------------------------------------
   Convert a UTF-8 character to a MultiByte character.
   Return the size of the converted character in bytes.
*/
int
ldap_x_utf8_to_mb ( char *mbchar, const char *utf8char,
		int (*f_wctomb)(char *mbchar, wchar_t wchar) )
{
	wchar_t wchar;
	int n;
	char tmp[6];				/* Large enough for biggest multibyte char */

	if (f_wctomb == NULL)		/* If no conversion function was given... */
		f_wctomb = wctomb;		/*    use the local ANSI C function */
 
	/* First convert UTF-8 char to a wide char */
	n = ldap_x_utf8_to_wc( &wchar, utf8char);

	if (n == -1)
		return -1;		/* Invalid UTF-8 character */

	if (mbchar == NULL)
		n = f_wctomb( tmp, wchar );
	else
		n = f_wctomb( mbchar, wchar);

	return n;
}

/*-----------------------------------------------------------------------------
   Convert a UTF-8 string to a MultiByte string.
   No more than 'count' bytes will be written to the output buffer.
   Return the size of the converted string in bytes, excl null terminator.
*/
int
ldap_x_utf8s_to_mbs ( char *mbstr, const char *utf8str, size_t count,
		size_t (*f_wcstombs)(char *mbstr, const wchar_t *wcstr, size_t count) )
{
	wchar_t *wcs;
	size_t wcsize;
    int n;

	if (f_wcstombs == NULL)		/* If no conversion function was given... */
		f_wcstombs = wcstombs;	/*    use the local ANSI C function */
 
	if (utf8str == NULL || *utf8str == 0)	/* NULL or empty input string */
	{
		if (mbstr)
			*mbstr = 0;
		return 0;
	}

/* Allocate memory for the maximum size wchar string that we could get. */
	wcsize = strlen(utf8str) + 1;
	wcs = (wchar_t *)LDAP_MALLOC(wcsize * sizeof(wchar_t));
	if (wcs == NULL)
		return -1;				/* Memory allocation failure. */

	/* First convert the UTF-8 string to a wide char string */
	n = ldap_x_utf8s_to_wcs( wcs, utf8str, wcsize);

	/* Then convert wide char string to multi-byte string */
	if (n != -1)
	{
		n = f_wcstombs(mbstr, wcs, count);
	}

	LDAP_FREE(wcs);

	return n;
}

/*-----------------------------------------------------------------------------
   Convert a MultiByte character to a UTF-8 character.
   'mbsize' indicates the number of bytes of 'mbchar' to check.
   Returns the number of bytes written to the output character.
*/
int
ldap_x_mb_to_utf8 ( char *utf8char, const char *mbchar, size_t mbsize,
		int (*f_mbtowc)(wchar_t *wchar, const char *mbchar, size_t count) )
{
    wchar_t wchar;
    int n;

	if (f_mbtowc == NULL)		/* If no conversion function was given... */
		f_mbtowc = mbtowc;		/*    use the local ANSI C function */
 
    if (mbsize == 0)				/* 0 is not valid. */
        return -1;

    if (mbchar == NULL || *mbchar == 0)
    {
        if (utf8char)
            *utf8char = 0;
        return 1;
    }

	/* First convert the MB char to a Wide Char */
	n = f_mbtowc( &wchar, mbchar, mbsize);

	if (n == -1)
		return -1;

	/* Convert the Wide Char to a UTF-8 character. */
	n = ldap_x_wc_to_utf8( utf8char, wchar, LDAP_MAX_UTF8_LEN);

	return n;
}


/*-----------------------------------------------------------------------------
   Convert a MultiByte string to a UTF-8 string.
   No more than 'count' bytes will be written to the output buffer.
   Return the size of the converted string in bytes, excl null terminator.
*/   
int
ldap_x_mbs_to_utf8s ( char *utf8str, const char *mbstr, size_t count,
		size_t (*f_mbstowcs)(wchar_t *wcstr, const char *mbstr, size_t count) )
{
	wchar_t *wcs;
	int n;
	size_t wcsize;

	if (mbstr == NULL)		   /* Treat NULL input string as an empty string */
		mbstr = "";

	if (f_mbstowcs == NULL)		/* If no conversion function was given... */
		f_mbstowcs = mbstowcs;	/*    use the local ANSI C function */
 
	/* Allocate memory for the maximum size wchar string that we could get. */
	wcsize = strlen(mbstr) + 1;
	wcs = (wchar_t *)LDAP_MALLOC( wcsize * sizeof(wchar_t) );
	if (wcs == NULL)
		return -1;

	/* First convert multi-byte string to a wide char string */
	n = f_mbstowcs(wcs, mbstr, wcsize);

	/* Convert wide char string to UTF-8 string */
	if (n != -1)
	{
		n = ldap_x_wcs_to_utf8s( utf8str, wcs, count);
	}

	LDAP_FREE(wcs);

	return n;	
}

#endif /* SIZEOF_WCHAR_T >= 4 */