Use ucdata (very experimental)

build/main.dsw

@@ -128,6 +128,9 @@ Package=<4>
     Begin Project Dependency
     Project_Dep_Name passwd
     End Project Dependency
+    Begin Project Dependency
+    Project_Dep_Name ucgendat
+    End Project Dependency
 }}}
 
 ###############################################################################
@@ -512,6 +515,9 @@ Package=<4>
     Begin Project Dependency
     Project_Dep_Name libslapd
     End Project Dependency
+    Begin Project Dependency
+    Project_Dep_Name liblunicode
+    End Project Dependency
 }}}
 
 ###############################################################################
@@ -548,6 +554,9 @@ Package=<4>
     Begin Project Dependency
     Project_Dep_Name backldbm
     End Project Dependency
+    Begin Project Dependency
+    Project_Dep_Name liblunicode
+    End Project Dependency
 }}}
 
 ###############################################################################
@@ -587,6 +596,9 @@ Package=<4>
     Begin Project Dependency
     Project_Dep_Name libslapd
     End Project Dependency
+    Begin Project Dependency
+    Project_Dep_Name liblunicode
+    End Project Dependency
 }}}
 
 ###############################################################################
@@ -623,6 +635,9 @@ Package=<4>
     Begin Project Dependency
     Project_Dep_Name libslapd
     End Project Dependency
+    Begin Project Dependency
+    Project_Dep_Name liblunicode
+    End Project Dependency
 }}}
 
 ###############################################################################
@@ -642,6 +657,18 @@ Package=<4>
 
 ###############################################################################
 
+Project: "ucgendat"=..\libraries\liblunicode\ucgetdat.dsp - Package Owner=<4>
+
+Package=<5>
+{{{
+}}}
+
+Package=<4>
+{{{
+}}}
+
+###############################################################################
+
 Project: "ud"=..\clients\ud\ud.dsp - Package Owner=<4>
 
 Package=<5>

libraries/liblunicode/liblunicode.dsp

@@ -128,6 +128,10 @@ LIB32=link.exe -lib
 # Name "liblunicode - Win32 Single Release"
 # Begin Source File
 
+SOURCE=..\..\include\ldap_pvt_uc.h
+# End Source File
+# Begin Source File
+
 SOURCE=.\ucdata\ucdata.c
 # End Source File
 # Begin Source File
@@ -136,6 +140,10 @@ SOURCE=.\ucdata\ucdata.h
 # End Source File
 # Begin Source File
 
+SOURCE=.\ucstr.c
+# End Source File
+# Begin Source File
+
 SOURCE=.\ure\ure.c
 # End Source File
 # Begin Source File

libraries/liblunicode/ucdata/ucdata.c

@@ -24,13 +24,7 @@
  * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#ifndef lint
+/* $Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $" */
-#ifdef __GNUC__
-static char rcsid[] __attribute__ ((unused)) = "$Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $";
-#else
-static char rcsid[] = "$Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $";
-#endif
-#endif
 
 #include "portable.h"
 
@@ -61,12 +55,14 @@ typedef struct {
  * A simple array of 32-bit masks for lookup.
  */
 static unsigned long masks32[32] = {
-    0x00000001, 0x00000002, 0x00000004, 0x00000008, 0x00000010, 0x00000020,
+	0x00000001UL, 0x00000002UL, 0x00000004UL, 0x00000008UL,
-    0x00000040, 0x00000080, 0x00000100, 0x00000200, 0x00000400, 0x00000800,
+	0x00000010UL, 0x00000020UL, 0x00000040UL, 0x00000080UL,
-    0x00001000, 0x00002000, 0x00004000, 0x00008000, 0x00010000, 0x00020000,
+	0x00000100UL, 0x00000200UL, 0x00000400UL, 0x00000800UL,
-    0x00040000, 0x00080000, 0x00100000, 0x00200000, 0x00400000, 0x00800000,
+	0x00001000UL, 0x00002000UL, 0x00004000UL, 0x00008000UL,
-    0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000,
+	0x00010000UL, 0x00020000UL, 0x00040000UL, 0x00080000UL,
-    0x40000000, 0x80000000
+	0x00100000UL, 0x00200000UL, 0x00400000UL, 0x00800000UL,
+	0x01000000UL, 0x02000000UL, 0x04000000UL, 0x08000000UL,
+	0x10000000UL, 0x20000000UL, 0x40000000UL, 0x80000000UL
 };
 
 #define endian_short(cc) (((cc) >> 8) | (((cc) & 0xff) << 8))

libraries/liblunicode/ucdata/ucgendat.c

@@ -1,3 +1,4 @@
+/* $OpenLDAP$
 /*
  * Copyright 1999 Computing Research Labs, New Mexico State University
  *
@@ -19,13 +20,7 @@
  * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#ifndef lint
+/* $Id: ucgendat.c,v 1.3 1999/10/07 20:49:56 mleisher Exp $" */
-#ifdef __GNUC__
-static char rcsid[] __attribute__ ((unused)) = "$Id: ucgendat.c,v 1.3 1999/10/07 20:49:56 mleisher Exp $";
-#else
-static char rcsid[] = "$Id: ucgendat.c,v 1.3 1999/10/07 20:49:56 mleisher Exp $";
-#endif
-#endif
 
 #include "portable.h"
 

libraries/liblunicode/ucgetdat.dsp

@@ -0,0 +1,145 @@
+# Microsoft Developer Studio Project File - Name="ucgendat" - Package Owner=<4>
+# Microsoft Developer Studio Generated Build File, Format Version 5.00
+# ** DO NOT EDIT **
+
+# TARGTYPE "Win32 (x86) Console Application" 0x0103
+
+CFG=ucgendat - Win32 Single Debug
+!MESSAGE This is not a valid makefile. To build this project using NMAKE,
+!MESSAGE use the Export Makefile command and run
+!MESSAGE 
+!MESSAGE NMAKE /f "ucgendat.mak".
+!MESSAGE 
+!MESSAGE You can specify a configuration when running NMAKE
+!MESSAGE by defining the macro CFG on the command line. For example:
+!MESSAGE 
+!MESSAGE NMAKE /f "ucgendat.mak" CFG="ucgendat - Win32 Single Debug"
+!MESSAGE 
+!MESSAGE Possible choices for configuration are:
+!MESSAGE 
+!MESSAGE "ucgendat - Win32 Release" (based on "Win32 (x86) Console Application")
+!MESSAGE "ucgendat - Win32 Debug" (based on "Win32 (x86) Console Application")
+!MESSAGE "ucgendat - Win32 Single Debug" (based on\
+ "Win32 (x86) Console Application")
+!MESSAGE "ucgendat - Win32 Single Release" (based on\
+ "Win32 (x86) Console Application")
+!MESSAGE 
+
+# Begin Project
+# PROP Scc_ProjName ""
+# PROP Scc_LocalPath ""
+CPP=cl.exe
+RSC=rc.exe
+
+!IF  "$(CFG)" == "ucgendat - Win32 Release"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 0
+# PROP BASE Output_Dir "Release"
+# PROP BASE Intermediate_Dir "Release"
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 0
+# PROP Output_Dir "..\..\Release"
+# PROP Intermediate_Dir "..\..\Release\ucgendat"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD CPP /nologo /MT /W3 /GX /O2 /I "..\..\include" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD BASE RSC /l 0x409 /d "NDEBUG"
+# ADD RSC /l 0x409 /d "NDEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386
+# ADD LINK32 oldap32.lib olber32.lib sasl.lib libsasl.lib ws2_32.lib /nologo /subsystem:console /machine:I386 /libpath:"..\..\Release"
+
+!ELSEIF  "$(CFG)" == "ucgendat - Win32 Debug"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 1
+# PROP BASE Output_Dir "ucgendat_"
+# PROP BASE Intermediate_Dir "ucgendat_"
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 1
+# PROP Output_Dir "..\..\Debug"
+# PROP Intermediate_Dir "..\..\Debug\ucgendat"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /W3 /Gm /GX /Zi /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD CPP /nologo /MTd /W3 /Gm /GX /Zi /Od /I "..\..\include" /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /FR /YX /FD /c
+# ADD BASE RSC /l 0x409 /d "_DEBUG"
+# ADD RSC /l 0x409 /d "_DEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept
+# ADD LINK32 oldap32.lib olber32.lib libsasl.lib ws2_32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept /libpath:"..\..\Debug"
+
+!ELSEIF  "$(CFG)" == "ucgendat - Win32 Single Debug"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 1
+# PROP BASE Output_Dir "ucgendat_"
+# PROP BASE Intermediate_Dir "ucgendat_"
+# PROP BASE Ignore_Export_Lib 0
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 1
+# PROP Output_Dir "..\..\SDebug"
+# PROP Intermediate_Dir "..\..\SDebug\ucgendat"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /W3 /Gm /GX /Zi /Od /I "..\..\include" /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD CPP /nologo /W3 /Gm /GX /Zi /Od /I "..\..\include" /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /FR /YX /FD /c
+# ADD BASE RSC /l 0x409 /d "_DEBUG"
+# ADD RSC /l 0x409 /d "_DEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 oldap32.lib olber32.lib ws2_32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept /libpath:"..\Debug"
+# ADD LINK32 oldap32.lib olber32.lib libsasl.lib ws2_32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept /libpath:"..\..\SDebug"
+
+!ELSEIF  "$(CFG)" == "ucgendat - Win32 Single Release"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 0
+# PROP BASE Output_Dir "ucgendat0"
+# PROP BASE Intermediate_Dir "ucgendat0"
+# PROP BASE Ignore_Export_Lib 0
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 0
+# PROP Output_Dir "..\..\SRelease"
+# PROP Intermediate_Dir "..\..\SRelease\ucgendat"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /W3 /GX /O2 /I "..\..\include" /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD CPP /nologo /W3 /GX /O2 /I "..\..\include" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /YX /FD /c
+# ADD BASE RSC /l 0x409 /d "NDEBUG"
+# ADD RSC /l 0x409 /d "NDEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 oldap32.lib olber32.lib ws2_32.lib /nologo /subsystem:console /machine:I386 /libpath:"..\Release"
+# ADD LINK32 oldap32.lib olber32.lib sasl.lib libsasl.lib ws2_32.lib /nologo /subsystem:console /machine:I386 /libpath:"..\..\SRelease"
+
+!ENDIF 
+
+# Begin Target
+
+# Name "ucgendat - Win32 Release"
+# Name "ucgendat - Win32 Debug"
+# Name "ucgendat - Win32 Single Debug"
+# Name "ucgendat - Win32 Single Release"
+# Begin Source File
+
+SOURCE=.\ucdata\ucgendat.c
+# End Source File
+# End Target
+# End Project

libraries/liblunicode/ucstr.c

@@ -0,0 +1,78 @@
+#include "portable.h"
+
+#include <ldap_pvt_uc.h>
+
+int ucstrncmp(
+	const ldap_unicode_t *u1,
+	const ldap_unicode_t *u2,
+	ber_len_t n )
+{
+	for(; 0 < n; ++u1, ++u2, --n ) {
+		if( *u1 != *u2 ) {
+			return *u1 < *u2 ? -1 : +1;
+		}
+		if ( *u1 == 0 ) {
+			return 0;
+		}
+	}
+	return 0;
+}
+
+int ucstrncasecmp(
+	const ldap_unicode_t *u1,
+	const ldap_unicode_t *u2,
+	ber_len_t n )
+{
+	for(; 0 < n; ++u1, ++u2, --n ) {
+		ldap_unicode_t uu1 = uctoupper( *u1 );
+		ldap_unicode_t uu2 = uctoupper( *u2 );
+
+		if( uu1 != uu2 ) {
+			return uu1 < uu2 ? -1 : +1;
+		}
+		if ( uu1 == 0 ) {
+			return 0;
+		}
+	}
+	return 0;
+}
+
+ldap_unicode_t * ucstrnchr(
+	const ldap_unicode_t *u,
+	ber_len_t n,
+	ldap_unicode_t c )
+{
+	for(; 0 < n; ++u, --n ) {
+		if( *u == c ) {
+			return (ldap_unicode_t *) u;
+		}
+	}
+
+	return NULL;
+}
+
+ldap_unicode_t * ucstrncasechr(
+	const ldap_unicode_t *u,
+	ber_len_t n,
+	ldap_unicode_t c )
+{
+	c = uctoupper( c );
+	for(; 0 < n; ++u, --n ) {
+		if( uctoupper( *u ) == c ) {
+			return (ldap_unicode_t *) u;
+		}
+	}
+
+	return NULL;
+}
+
+void ucstr2upper(
+	ldap_unicode_t *u,
+	ber_len_t n )
+{
+	for(; 0 < n; ++u, --n ) {
+		*u = uctoupper( *u );
+	}
+}
+
+

libraries/liblunicode/ure/ure.c

@@ -25,9 +25,7 @@
  * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#ifndef lint
+/* $Id: ure.c,v 1.2 1999/09/21 15:47:43 mleisher Exp $" */
-static char rcsid[] = "$Id: ure.c,v 1.2 1999/09/21 15:47:43 mleisher Exp $";
-#endif
 
 #include "portable.h"
 

libraries/liblunicode/ure/urestubs.c

@@ -25,12 +25,16 @@
  * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#ifndef lint
+/* $Id: urestubs.c,v 1.2 1999/09/21 15:47:44 mleisher Exp $" */
-static char rcsid[] = "$Id: urestubs.c,v 1.2 1999/09/21 15:47:44 mleisher Exp $";
-#endif
 
 #include "ure.h"
 
+#ifdef _MSC_VER
+#  include "../ucdata/ucdata.h"
+#else
+#  include "ucdata.h"
+#endif
+
 /*
  * This file contains stub routines needed by the URE package to test
  * character properties and other Unicode implementation specific details.
@@ -42,9 +46,53 @@ static char rcsid[] = "$Id: urestubs.c,v 1.2 1999/09/21 15:47:44 mleisher Exp $"
  */
 ucs4_t _ure_tolower(ucs4_t c)
 {
-    return c;
+    return uctoupper(c);
 }
 
+static struct maskmap {
+	unsigned long mask1;
+	unsigned long mask2;
+} masks[32] = {
+	{ UC_MN, 0 },	/* _URE_NONSPACING */
+	{ UC_MC, 0 },	/* _URE_COMBINING */
+	{ UC_ND, 0 },	/* _URE_NUMDIGIT */
+	{ UC_NL|UC_NO, 0 },	/* _URE_NUMOTHER */
+	{ UC_ZS, 0 },	/* _URE_SPACESEP */
+	{ UC_ZL, 0 },	/* _URE_LINESEP */
+	{ UC_ZP, 0 },	/* _URE_PARASEP */
+	{ UC_CC, 0 },	/* _URE_CNTRL */
+	{ UC_CO, 0 },	/* _URE_PUA */
+
+	{ UC_LU, 0 },	/* _URE_UPPER */
+	{ UC_LL, 0 },	/* _URE_LOWER */
+	{ UC_LT, 0 },	/* _URE_TITLE */
+	{ UC_LM, 0 },	/* _URE_MODIFIER */
+	{ UC_LO, 0 },	/* _URE_OTHERLETTER */
+	{ UC_PD, 0 },	/* _URE_DASHPUNCT */
+	{ UC_PS, 0 },	/* _URE_OPENPUNCT */
+	{ UC_PC, 0 },	/* _URE_CLOSEPUNCT */
+	{ UC_PO, 0 },	/* _URE_OTHERPUNCT */
+	{ UC_SM, 0 },	/* _URE_MATHSYM */
+	{ UC_SC, 0 },	/* _URE_CURRENCYSYM */
+	{ UC_SO, 0 },	/* _URE_OTHERSYM */
+
+	{ UC_L, 0 },	/* _URE_LTR */
+	{ UC_R, 0 },	/* _URE_RTL */
+
+	{ 0, UC_EN },	/* _URE_EURONUM */
+	{ 0, UC_ES },	/* _URE_EURONUMSEP */
+	{ 0, UC_ET },	/* _URE_EURONUMTERM */
+	{ 0, UC_AN },	/* _URE_ARABNUM */
+	{ 0, UC_CS },	/* _URE_COMMONSEP */
+
+	{ 0, UC_B },	/* _URE_BLOCKSEP */
+	{ 0, UC_S },	/* _URE_SEGMENTSEP */
+
+	{ 0, UC_WS },	/* _URE_WHITESPACE */
+	{ 0, UC_ON }	/* _URE_OTHERNEUT */
+};
+
+
 /*
  * This routine takes a set of URE character property flags (see ure.h) along
  * with a character and tests to see if the character has one or more of those
@@ -53,5 +101,15 @@ ucs4_t _ure_tolower(ucs4_t c)
 int
 _ure_matches_properties(unsigned long props, ucs4_t c)
 {
-    return 1;
+	int i;
+	unsigned long mask1=0, mask2=0;
+
+	for( i=0; i<32; i++ ) {
+		if( props & (1 << i) ) {
+			mask1 |= masks[i].mask1;
+			mask2 |= masks[i].mask2;
+		}
+	}
+
+	return ucisprop( mask1, mask2, c );
 }

servers/slapd/schema_init.c

@@ -15,6 +15,7 @@
 
 #include "slap.h"
 #include "ldap_pvt.h"
+#include "ldap_pvt_uc.h"
 #include "lutil_md5.h"
 
 /* recycled validatation routines */
@@ -46,7 +47,7 @@
 #define objectIdentifierMatch			numericStringMatch
 #define integerMatch					numericStringMatch
 #define telephoneNumberMatch			numericStringMatch
-#define telephoneNumberSubstringsMatch	caseIgnoreIA5SubstringsMatch
+#define telephoneNumberSubstringsMatch	caseIgnoreSubstringsMatch
 #define generalizedTimeMatch			numericStringMatch
 #define generalizedTimeOrderingMatch	numericStringMatch
 #define uniqueMemberMatch				dnMatch
@@ -348,62 +349,6 @@ booleanMatch(
 	return LDAP_SUCCESS;
 }
 
-#if 0
-static int
-UTF8casecmp(
-	struct berval *right,
-	struct berval *left )
-{
-	ber_len_t r, l;
-	int rlen, llen;
-	ldap_unicode_t ru, lu;
-	ldap_unicode_t ruu, luu;
-
-	for( r=0, l=0;
-		r < right->bv_len && l < left->bv_len;
-		r+=rlen, l+=llen )
-	{
-		/*
-		 * XXYYZ: we convert to ucs4 even though -llunicode
-		 * expects ucs2 in an unsigned long
-		 */
-		ru = ldap_utf8_to_ucs4( &right->bv_val[r] );
-		if( ru == LDAP_UCS4_INVALID ) {
-			return 1;
-		}
-
-		lu = ldap_utf8_to_ucs4( &left->bv_val[l] );
-		if( lu == LDAP_UCS4_INVALID ) {
-			return -1;
-		}
-
-		ruu = uctoupper( ru );
-		luu = uctoupper( lu );
-
-		if( ruu > luu ) {
-			return 1;
-		} else if( luu > ruu ) {
-			return -1;
-		}
-
-		rlen = LDAP_UTF8_CHARLEN( &right->bv_val[r] );
-		llen = LDAP_UTF8_CHARLEN( &left->bv_val[l] );
-	}
-
-	if( r < right->bv_len ) {
-		/* less left */
-		return -1;
-	}
-
-	if( l < left->bv_len ) {
-		/* less right */
-		return 1;
-	}
-
-	return 0;
-}
-#endif
-
 static int
 UTF8StringValidate(
 	Syntax *syntax,
@@ -432,80 +377,84 @@ UTF8StringValidate(
 	return LDAP_SUCCESS;
 }
 
+#define UNICODE2LEN(n)	((n)*sizeof(ldap_unicode_t))
+#define LEN2UNICODE(n)	((n)/sizeof(ldap_unicode_t))
+
+#define SLAP_INDEX_SUBSTR_UMINLEN	UNICODE2LEN(SLAP_INDEX_SUBSTR_MINLEN)
+#define SLAP_INDEX_SUBSTR_UMAXLEN	UNICODE2LEN(SLAP_INDEX_SUBSTR_MAXLEN)
+#define SLAP_INDEX_SUBSTR_USTEP		UNICODE2LEN(SLAP_INDEX_SUBSTR_STEP)
+
 static int
 UTF8StringNormalize(
 	Syntax *syntax,
 	struct berval *val,
 	struct berval **normalized )
 {
+	ber_len_t bcount;
+	ber_len_t ucount;
+	int len, space;
+	char *u;
 	struct berval *newval;
-	char *p, *q, *s;
+	ldap_unicode_t *uc;
 
-	newval = ch_malloc( sizeof( struct berval ) );
+	if( !val->bv_len ) return LDAP_INVALID_SYNTAX;
 
-	p = val->bv_val;
+	u = val->bv_val;
+	ucount = 0;
+	space = 1;
 
-	/* Ignore initial whitespace */
+	for( bcount = val->bv_len; bcount > 0; bcount-=len, u+=len ) {
-	while ( ldap_utf8_isspace( p ) ) {
+		/* get the length indicated by the first byte */
-		LDAP_UTF8_INCR( p );
+		len = LDAP_UTF8_CHARLEN( u );
+
+		/* should not be zero */
+		if( len == 0 ) return LDAP_INVALID_SYNTAX;
+
+		if( ldap_utf8_isspace( u ) ) {
+			if( space ) continue;
+		} else {
+			space=1;
+		}
+
+		ucount++;
 	}
 
-	if( *p == '\0' ) {
+	if( ucount <= 1 ) {
-		ch_free( newval );
 		return LDAP_INVALID_SYNTAX;
 	}
 
-	newval->bv_val = ch_strdup( p );
+	if( space ) {
-	p = q = newval->bv_val;
+		ucount--;
-	s = NULL;
+	}
 
-	while ( *p ) {
+	newval = ch_malloc( sizeof( struct berval ) );
-		int len;
+	newval->bv_val = ch_malloc( UNICODE2LEN(ucount+1) );
+	uc = (ldap_unicode_t *) newval->bv_val;
 
-		if ( ldap_utf8_isspace( p ) ) {
+	u = val->bv_val;
-			len = LDAP_UTF8_COPY(q,p);
+	ucount = 0;
-			s=q;
+	space = 1;
-			p+=len;
-			q+=len;
 
-			/* Ignore the extra whitespace */
+	for( bcount = val->bv_len; bcount > 0; bcount-=len, u+=len ) {
-			while ( ldap_utf8_isspace( p ) ) {
+		/* get the length indicated by the first byte */
-				LDAP_UTF8_INCR( p );
+		len = LDAP_UTF8_CHARLEN( u );
-			}
+
+		/* should not be zero */
+		if( len == 0 ) return LDAP_INVALID_SYNTAX;
+
+		if( ldap_utf8_isspace( u ) ) {
+			if( space ) continue;
 		} else {
-			len = LDAP_UTF8_COPY(q,p);
+			space=1;
-			s=NULL;
-			p+=len;
-			q+=len;
 		}
+
+		uc[ucount++] = ldap_utf8_to_unicode( u );
 	}
 
-	assert( *newval->bv_val );
+	if( space ) ucount--;
-	assert( newval->bv_val < p );
+	uc[ucount] = 0;
-	assert( p >= q );
-
-	/* cannot start with a space */
-	assert( !ldap_utf8_isspace(newval->bv_val) );
-
-	/*
-	 * If the string ended in space, backup the pointer one
-	 * position.  One is enough because the above loop collapsed
-	 * all whitespace to a single space.
-	 */
-
-	if ( s != NULL ) {
-		q = s;
-	}
-
-	/* cannot end with a space */
-	assert( !ldap_utf8_isspace( LDAP_UTF8_PREV(q) ) );
-
-	/* null terminate */
-	*q = '\0';
-
-	newval->bv_len = q - newval->bv_val;
-	*normalized = newval;
 
+	newval->bv_len = UNICODE2LEN(ucount);
 	return LDAP_SUCCESS;
 }
 
@@ -521,9 +470,9 @@ caseExactMatch(
 	int match = value->bv_len - ((struct berval *) assertedValue)->bv_len;
 
 	if( match == 0 ) {
-		match = strncmp( value->bv_val,
+		match = ucstrncmp( (ldap_unicode_t *) value->bv_val,
-			((struct berval *) assertedValue)->bv_val,
+			(ldap_unicode_t *) ((struct berval *) assertedValue)->bv_val,
-			value->bv_len );
+			LEN2UNICODE(value->bv_len) );
 	}
 
 	*matchp = match;
@@ -564,8 +513,10 @@ caseExactSubstringsMatch(
 			goto done;
 		}
 
-		match = strncmp( sub->sa_initial->bv_val, left.bv_val,
+		match = ucstrncmp(
-			sub->sa_initial->bv_len );
+			(ldap_unicode_t *) sub->sa_initial->bv_val,
+			(ldap_unicode_t *) left.bv_val,
+			LEN2UNICODE(sub->sa_initial->bv_len) );
 
 		if( match != 0 ) {
 			goto done;
@@ -582,9 +533,10 @@ caseExactSubstringsMatch(
 			goto done;
 		}
 
-		match = strncmp( sub->sa_final->bv_val,
+		match = ucstrncmp(
-			&left.bv_val[left.bv_len - sub->sa_final->bv_len],
+			(ldap_unicode_t *) sub->sa_final->bv_val,
-			sub->sa_final->bv_len );
+			(ldap_unicode_t *) &left.bv_val[left.bv_len - sub->sa_final->bv_len],
+			LEN2UNICODE(sub->sa_final->bv_len) );
 
 		if( match != 0 ) {
 			goto done;
@@ -610,7 +562,11 @@ retry:
 				continue;
 			}
 
-			p = strchr( left.bv_val, *sub->sa_any[i]->bv_val );
+			p = (char *) ucstrnchr(
+				(ldap_unicode_t *) left.bv_val,
+				LEN2UNICODE(left.bv_len),
+				((ldap_unicode_t *) sub->sa_any[i]->bv_val)[0] );
+
 
 			if( p == NULL ) {
 				match = 1;
@@ -634,13 +590,14 @@ retry:
 				goto done;
 			}
 
-			match = strncmp( left.bv_val,
+			match = ucstrncmp(
-				sub->sa_any[i]->bv_val,
+				(ldap_unicode_t *) left.bv_val,
-				sub->sa_any[i]->bv_len );
+				(ldap_unicode_t *) sub->sa_any[i]->bv_val,
+				LEN2UNICODE(sub->sa_any[i]->bv_len) );
 
 			if( match != 0 ) {
-				left.bv_val++;
+				left.bv_val += UNICODE2LEN(1);
-				left.bv_len--;
+				left.bv_len -= UNICODE2LEN(1);
 				goto retry;
 			}
 
@@ -777,31 +734,34 @@ int caseExactSubstringsIndexer(
 	nkeys=0;
 	for( i=0; values[i] != NULL; i++ ) {
 		/* count number of indices to generate */
-		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_MINLEN ) {
+		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_UMINLEN ) {
 			continue;
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
-					( SLAP_INDEX_SUBSTR_MINLEN - 1);
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			} else {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MINLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			}
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_ANY ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
 			}
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_FINAL ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
-					( SLAP_INDEX_SUBSTR_MINLEN - 1);
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			} else {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MINLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			}
 		}
 	}
@@ -824,21 +784,21 @@ int caseExactSubstringsIndexer(
 		struct berval *value;
 
 		value = values[i];
-		if( value->bv_len < SLAP_INDEX_SUBSTR_MINLEN ) continue;
+		if( value->bv_len < SLAP_INDEX_SUBSTR_UMINLEN ) continue;
 
 		if( ( flags & SLAP_INDEX_SUBSTR_ANY ) &&
-			( value->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) )
+			( value->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) )
 		{
 			char pre = SLAP_INDEX_SUBSTR_PREFIX;
-			max = value->bv_len - ( SLAP_INDEX_SUBSTR_MAXLEN - 1);
+			max = value->bv_len -
+				( SLAP_INDEX_SUBSTR_UMAXLEN - UNICODE2LEN(1));
 
-			for( j=0; j<max; j++ ) {
+			for( j=0; j<max; j+=UNICODE2LEN(1) ) {
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
 					lutil_MD5Update( &MD5context,
 						prefix->bv_val, prefix->bv_len );
 				}
-
 				lutil_MD5Update( &MD5context,
 					&pre, sizeof( pre ) );
 				lutil_MD5Update( &MD5context,
@@ -846,18 +806,17 @@ int caseExactSubstringsIndexer(
 				lutil_MD5Update( &MD5context,
 					mr->smr_oid, mlen );
 				lutil_MD5Update( &MD5context,
-					&value->bv_val[j],
+					&value->bv_val[j], SLAP_INDEX_SUBSTR_UMAXLEN );
-					SLAP_INDEX_SUBSTR_MAXLEN );
 				lutil_MD5Final( MD5digest, &MD5context );
 
 				keys[nkeys++] = ber_bvdup( &digest );
 			}
 		}
 
-		max = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		max = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
-		for( j=SLAP_INDEX_SUBSTR_MINLEN; j<=max; j++ ) {
+		for( j=SLAP_INDEX_SUBSTR_UMINLEN; j<=max; j+=UNICODE2LEN(1) ) {
 			char pre;
 
 			if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
@@ -934,7 +893,7 @@ int caseExactSubstringsFilter(
 	struct berval digest;
 
 	if( flags & SLAP_INDEX_SUBSTR_INITIAL && sa->sa_initial != NULL &&
-		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		nkeys++;
 	}
@@ -942,16 +901,16 @@ int caseExactSubstringsFilter(
 	if( flags & SLAP_INDEX_SUBSTR_ANY && sa->sa_any != NULL ) {
 		ber_len_t i;
 		for( i=0; sa->sa_any[i] != NULL; i++ ) {
-			if( sa->sa_any[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( sa->sa_any[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				/* don't bother accounting for stepping */
-				nkeys += sa->sa_any[i]->bv_len -
+				nkeys += LEN2UNICODE(sa->sa_any[i]->bv_len) -
 					( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
 			}
 		}
 	}
 
 	if( flags & SLAP_INDEX_SUBSTR_FINAL && sa->sa_final != NULL &&
-		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		nkeys++;
 	}
@@ -971,13 +930,13 @@ int caseExactSubstringsFilter(
 	nkeys = 0;
 
 	if( flags & SLAP_INDEX_SUBSTR_INITIAL && sa->sa_initial != NULL &&
-		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		pre = SLAP_INDEX_SUBSTR_INITIAL_PREFIX;
 		value = sa->sa_initial;
 
-		klen = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
 		lutil_MD5Init( &MD5context );
 		if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1000,18 +959,18 @@ int caseExactSubstringsFilter(
 	if( flags & SLAP_INDEX_SUBSTR_ANY && sa->sa_any != NULL ) {
 		ber_len_t i, j;
 		pre = SLAP_INDEX_SUBSTR_PREFIX;
-		klen = SLAP_INDEX_SUBSTR_MAXLEN;
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN;
 
 		for( i=0; sa->sa_any[i] != NULL; i++ ) {
-			if( sa->sa_any[i]->bv_len < SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( sa->sa_any[i]->bv_len < SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				continue;
 			}
 
 			value = sa->sa_any[i];
 
 			for(j=0;
-				j <= value->bv_len - SLAP_INDEX_SUBSTR_MAXLEN;
+				j <= value->bv_len - SLAP_INDEX_SUBSTR_UMAXLEN;
-				j += SLAP_INDEX_SUBSTR_STEP )
+				j += SLAP_INDEX_SUBSTR_USTEP )
 			{
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1025,7 +984,7 @@ int caseExactSubstringsFilter(
 				lutil_MD5Update( &MD5context,
 					mr->smr_oid, mlen );
 				lutil_MD5Update( &MD5context,
-					&value->bv_val[j], klen ); 
+					&value->bv_val[j], klen );
 				lutil_MD5Final( MD5digest, &MD5context );
 
 				keys[nkeys++] = ber_bvdup( &digest );
@@ -1034,13 +993,13 @@ int caseExactSubstringsFilter(
 	}
 
 	if( flags & SLAP_INDEX_SUBSTR_FINAL && sa->sa_final != NULL &&
-		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		pre = SLAP_INDEX_SUBSTR_FINAL_PREFIX;
 		value = sa->sa_final;
 
-		klen = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
 		lutil_MD5Init( &MD5context );
 		if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1080,19 +1039,16 @@ caseIgnoreMatch(
 	struct berval *value,
 	void *assertedValue )
 {
-#if 0
-	*matchp = UTF8casecmp( value, (struct berval *) assertedValue );
-#else
 	int match = value->bv_len - ((struct berval *) assertedValue)->bv_len;
 
 	if( match == 0 ) {
-		match = strncasecmp( value->bv_val,
+		match = ucstrncasecmp(
-			((struct berval *) assertedValue)->bv_val,
+			(ldap_unicode_t *) value->bv_val,
-			value->bv_len );
+			(ldap_unicode_t *) ((struct berval *) assertedValue)->bv_val,
+			LEN2UNICODE(value->bv_len) );
 	}
 
 	*matchp = match;
-#endif
 	return LDAP_SUCCESS;
 }
 
@@ -1130,8 +1086,10 @@ caseIgnoreSubstringsMatch(
 			goto done;
 		}
 
-		match = strncasecmp( sub->sa_initial->bv_val, left.bv_val,
+		match = ucstrncasecmp(
-			sub->sa_initial->bv_len );
+			(ldap_unicode_t *) sub->sa_initial->bv_val,
+			(ldap_unicode_t *) left.bv_val,
+			LEN2UNICODE(sub->sa_initial->bv_len) );
 
 		if( match != 0 ) {
 			goto done;
@@ -1148,9 +1106,10 @@ caseIgnoreSubstringsMatch(
 			goto done;
 		}
 
-		match = strncasecmp( sub->sa_final->bv_val,
+		match = ucstrncasecmp(
-			&left.bv_val[left.bv_len - sub->sa_final->bv_len],
+			(ldap_unicode_t *) sub->sa_final->bv_val,
-			sub->sa_final->bv_len );
+			(ldap_unicode_t *) &left.bv_val[left.bv_len - sub->sa_final->bv_len],
+			LEN2UNICODE(sub->sa_final->bv_len) );
 
 		if( match != 0 ) {
 			goto done;
@@ -1176,7 +1135,10 @@ retry:
 				continue;
 			}
 
-			p = strcasechr( left.bv_val, *sub->sa_any[i]->bv_val );
+			p = (char *) ucstrncasechr(
+				(ldap_unicode_t *) left.bv_val,
+				LEN2UNICODE(left.bv_len),
+				((ldap_unicode_t *) sub->sa_any[i]->bv_val)[0] );
 
 			if( p == NULL ) {
 				match = 1;
@@ -1200,13 +1162,14 @@ retry:
 				goto done;
 			}
 
-			match = strncasecmp( left.bv_val,
+			match = ucstrncasecmp(
-				sub->sa_any[i]->bv_val,
+				(ldap_unicode_t *) left.bv_val,
-				sub->sa_any[i]->bv_len );
+				(ldap_unicode_t *) sub->sa_any[i]->bv_val,
+				LEN2UNICODE(sub->sa_any[i]->bv_len) );
 
 			if( match != 0 ) {
-				left.bv_val++;
+				left.bv_val += UNICODE2LEN(1);
-				left.bv_len--;
+				left.bv_len -= UNICODE2LEN(1);
 
 				goto retry;
 			}
@@ -1254,7 +1217,8 @@ int caseIgnoreIndexer(
 
 	for( i=0; values[i] != NULL; i++ ) {
 		struct berval *value = ber_bvdup( values[i] );
-		ldap_pvt_str2upper( value->bv_val );
+		ucstr2upper( (ldap_unicode_t *) value->bv_val,
+			LEN2UNICODE(value->bv_len) );
 
 		lutil_MD5Init( &MD5context );
 		if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1270,7 +1234,6 @@ int caseIgnoreIndexer(
 		lutil_MD5Final( MD5digest, &MD5context );
 
 		ber_bvfree( value );
-
 		keys[i] = ber_bvdup( &digest );
 	}
 
@@ -1302,7 +1265,8 @@ int caseIgnoreFilter(
 	mlen = strlen( mr->smr_oid );
 
 	value = ber_bvdup( (struct berval *) assertValue );
-	ldap_pvt_str2upper( value->bv_val );
+	ucstr2upper( (ldap_unicode_t *) value->bv_val,
+		LEN2UNICODE(value->bv_len) );
 
 	keys = ch_malloc( sizeof( struct berval * ) * 2 );
 
@@ -1351,31 +1315,34 @@ int caseIgnoreSubstringsIndexer(
 	nkeys=0;
 	for( i=0; values[i] != NULL; i++ ) {
 		/* count number of indices to generate */
-		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_MINLEN ) {
+		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_UMINLEN ) {
 			continue;
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
 					( SLAP_INDEX_SUBSTR_MINLEN - 1);
 			} else {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MINLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			}
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_ANY ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
 			}
 		}
 
 		if( flags & SLAP_INDEX_SUBSTR_FINAL ) {
-			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( values[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				nkeys += SLAP_INDEX_SUBSTR_MAXLEN -
 					( SLAP_INDEX_SUBSTR_MINLEN - 1);
 			} else {
-				nkeys += values[i]->bv_len - ( SLAP_INDEX_SUBSTR_MINLEN - 1 );
+				nkeys += LEN2UNICODE(values[i]->bv_len) -
+					( SLAP_INDEX_SUBSTR_MINLEN - 1 );
 			}
 		}
 	}
@@ -1397,24 +1364,25 @@ int caseIgnoreSubstringsIndexer(
 		int j,max;
 		struct berval *value;
 
-		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_MINLEN ) continue;
+		if( values[i]->bv_len < SLAP_INDEX_SUBSTR_UMINLEN ) continue;
 
 		value = ber_bvdup( values[i] );
-		ldap_pvt_str2upper( value->bv_val );
+		ucstr2upper( (ldap_unicode_t *) value->bv_val,
+			LEN2UNICODE(value->bv_len) );
 
 		if( ( flags & SLAP_INDEX_SUBSTR_ANY ) &&
-			( value->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) )
+			( value->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) )
 		{
 			char pre = SLAP_INDEX_SUBSTR_PREFIX;
-			max = value->bv_len - ( SLAP_INDEX_SUBSTR_MAXLEN - 1);
+			max = value->bv_len -
+				( SLAP_INDEX_SUBSTR_UMAXLEN - UNICODE2LEN(1));
 
-			for( j=0; j<max; j++ ) {
+			for( j=0; j<max; j+=UNICODE2LEN(1) ) {
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
 					lutil_MD5Update( &MD5context,
 						prefix->bv_val, prefix->bv_len );
 				}
-
 				lutil_MD5Update( &MD5context,
 					&pre, sizeof( pre ) );
 				lutil_MD5Update( &MD5context,
@@ -1423,17 +1391,17 @@ int caseIgnoreSubstringsIndexer(
 					mr->smr_oid, mlen );
 				lutil_MD5Update( &MD5context,
 					&value->bv_val[j],
-					SLAP_INDEX_SUBSTR_MAXLEN );
+					SLAP_INDEX_SUBSTR_UMAXLEN );
 				lutil_MD5Final( MD5digest, &MD5context );
 
 				keys[nkeys++] = ber_bvdup( &digest );
 			}
 		}
 
-		max = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		max = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
-		for( j=SLAP_INDEX_SUBSTR_MINLEN; j<=max; j++ ) {
+		for( j=SLAP_INDEX_SUBSTR_UMINLEN; j<=max; j+=UNICODE2LEN(1) ) {
 			char pre;
 
 			if( flags & SLAP_INDEX_SUBSTR_INITIAL ) {
@@ -1512,7 +1480,7 @@ int caseIgnoreSubstringsFilter(
 	struct berval digest;
 
 	if((flags & SLAP_INDEX_SUBSTR_INITIAL) && sa->sa_initial != NULL &&
-		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		nkeys++;
 	}
@@ -1520,16 +1488,16 @@ int caseIgnoreSubstringsFilter(
 	if((flags & SLAP_INDEX_SUBSTR_ANY) && sa->sa_any != NULL ) {
 		ber_len_t i;
 		for( i=0; sa->sa_any[i] != NULL; i++ ) {
-			if( sa->sa_any[i]->bv_len >= SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( sa->sa_any[i]->bv_len >= SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				/* don't bother accounting for stepping */
-				nkeys += sa->sa_any[i]->bv_len -
+				nkeys += LEN2UNICODE(sa->sa_any[i]->bv_len) -
 					( SLAP_INDEX_SUBSTR_MAXLEN - 1 );
 			}
 		}
 	}
 
 	if((flags & SLAP_INDEX_SUBSTR_FINAL) && sa->sa_final != NULL &&
-		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		nkeys++;
 	}
@@ -1549,14 +1517,15 @@ int caseIgnoreSubstringsFilter(
 	nkeys = 0;
 
 	if((flags & SLAP_INDEX_SUBSTR_INITIAL) && sa->sa_initial != NULL &&
-		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_initial->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		pre = SLAP_INDEX_SUBSTR_INITIAL_PREFIX;
 		value = ber_bvdup( sa->sa_initial );
-		ldap_pvt_str2upper( value->bv_val );
+		ucstr2upper( (ldap_unicode_t *) value->bv_val,
+			LEN2UNICODE(value->bv_len) );
 
-		klen = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
 		lutil_MD5Init( &MD5context );
 		if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1580,19 +1549,20 @@ int caseIgnoreSubstringsFilter(
 	if((flags & SLAP_INDEX_SUBSTR_ANY) && sa->sa_any != NULL ) {
 		ber_len_t i, j;
 		pre = SLAP_INDEX_SUBSTR_PREFIX;
-		klen = SLAP_INDEX_SUBSTR_MAXLEN;
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN;
 
 		for( i=0; sa->sa_any[i] != NULL; i++ ) {
-			if( sa->sa_any[i]->bv_len < SLAP_INDEX_SUBSTR_MAXLEN ) {
+			if( sa->sa_any[i]->bv_len < SLAP_INDEX_SUBSTR_UMAXLEN ) {
 				continue;
 			}
 
 			value = ber_bvdup( sa->sa_any[i] );
-			ldap_pvt_str2upper( value->bv_val );
+			ucstr2upper( (ldap_unicode_t *) value->bv_val,
+				LEN2UNICODE(value->bv_len) );
 
 			for(j=0;
-				j <= value->bv_len - SLAP_INDEX_SUBSTR_MAXLEN;
+				j <= value->bv_len - SLAP_INDEX_SUBSTR_UMAXLEN;
-				j += SLAP_INDEX_SUBSTR_STEP )
+				j += SLAP_INDEX_SUBSTR_USTEP )
 			{
 				lutil_MD5Init( &MD5context );
 				if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -1617,14 +1587,15 @@ int caseIgnoreSubstringsFilter(
 	}
 
 	if((flags & SLAP_INDEX_SUBSTR_FINAL) && sa->sa_final != NULL &&
-		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_MINLEN )
+		sa->sa_final->bv_len >= SLAP_INDEX_SUBSTR_UMINLEN )
 	{
 		pre = SLAP_INDEX_SUBSTR_FINAL_PREFIX;
 		value = ber_bvdup( sa->sa_final );
-		ldap_pvt_str2upper( value->bv_val );
+		ucstr2upper( (ldap_unicode_t *) value->bv_val,
+			LEN2UNICODE(value->bv_len) );
 
-		klen = SLAP_INDEX_SUBSTR_MAXLEN < value->bv_len
+		klen = SLAP_INDEX_SUBSTR_UMAXLEN < value->bv_len
-			? SLAP_INDEX_SUBSTR_MAXLEN : value->bv_len;
+			? SLAP_INDEX_SUBSTR_UMAXLEN : value->bv_len;
 
 		lutil_MD5Init( &MD5context );
 		if( prefix != NULL && prefix->bv_len > 0 ) {
@@ -2164,7 +2135,6 @@ int caseExactIA5SubstringsIndexer(
 					lutil_MD5Update( &MD5context,
 						prefix->bv_val, prefix->bv_len );
 				}
-
 				lutil_MD5Update( &MD5context,
 					&pre, sizeof( pre ) );
 				lutil_MD5Update( &MD5context,
@@ -2736,7 +2706,6 @@ int caseIgnoreIA5SubstringsIndexer(
 					lutil_MD5Update( &MD5context,
 						prefix->bv_val, prefix->bv_len );
 				}
-
 				lutil_MD5Update( &MD5context,
 					&pre, sizeof( pre ) );
 				lutil_MD5Update( &MD5context,

servers/slapd/schema_prep.c

@@ -15,7 +15,7 @@
 
 #include "slap.h"
 #include "ldap_pvt.h"
-#include "../libraries/liblunicode/ucdata.h"
+#include "ldap_pvt_uc.h"
 
 int schema_init_done = 0;