mirror of
https://git.openldap.org/openldap/openldap.git
synced 2025-02-05 13:40:08 +08:00
Final UCData-2.5 update
This commit is contained in:
Stig Venaas
parent
aa93a92313
commit
6c2ea5ec24
@ -4,7 +4,7 @@
|
||||
* COPYING RESTRICTIONS APPLY, see COPYRIGHT file
|
||||
*/
|
||||
/*
|
||||
* Copyright 1999 Computing Research Labs, New Mexico State University
|
||||
* Copyright 2001 Computing Research Labs, New Mexico State University
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
@ -24,7 +24,7 @@
|
||||
* OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
|
||||
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
/* $Id: ucdata.c,v 1.3 1999/08/23 16:14:09 mleisher Exp $" */
|
||||
/* $Id: ucdata.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
|
||||
|
||||
#include "portable.h"
|
||||
|
||||
@ -469,6 +469,193 @@ uctotitle(unsigned long code)
|
||||
return _uccase_lookup(code, l, r, field);
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Support for compositions.
|
||||
*
|
||||
**************************************************************************/
|
||||
|
||||
static unsigned long _uccomp_size;
|
||||
static unsigned long *_uccomp_data;
|
||||
|
||||
/*
|
||||
* Return -1 on error, 0 if okay
|
||||
*/
|
||||
static int
|
||||
_uccomp_load(char *paths, int reload)
|
||||
{
|
||||
FILE *in;
|
||||
unsigned long size, i;
|
||||
_ucheader_t hdr;
|
||||
|
||||
if (_uccomp_size > 0) {
|
||||
if (!reload)
|
||||
/*
|
||||
* The compositions have already been loaded.
|
||||
*/
|
||||
return 0;
|
||||
|
||||
free((char *) _uccomp_data);
|
||||
_uccomp_size = 0;
|
||||
}
|
||||
|
||||
if ((in = _ucopenfile(paths, "comp.dat", "rb")) == 0)
|
||||
return -1;
|
||||
|
||||
/*
|
||||
* Load the header.
|
||||
*/
|
||||
fread((char *) &hdr, sizeof(_ucheader_t), 1, in);
|
||||
|
||||
if (hdr.bom == 0xfffe) {
|
||||
hdr.cnt = endian_short(hdr.cnt);
|
||||
hdr.size.bytes = endian_long(hdr.size.bytes);
|
||||
}
|
||||
|
||||
_uccomp_size = hdr.cnt;
|
||||
_uccomp_data = (unsigned long *) malloc(hdr.size.bytes);
|
||||
|
||||
/*
|
||||
* Read the composition data in.
|
||||
*/
|
||||
size = hdr.size.bytes / sizeof(unsigned long);
|
||||
fread((char *) _uccomp_data, sizeof(unsigned long), size, in);
|
||||
|
||||
/*
|
||||
* Do an endian swap if necessary.
|
||||
*/
|
||||
if (hdr.bom == 0xfffe) {
|
||||
for (i = 0; i < size; i++)
|
||||
_uccomp_data[i] = endian_long(_uccomp_data[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Assume that the data is ordered on count, so that all compositions
|
||||
* of length 2 come first. Only handling length 2 for now.
|
||||
*/
|
||||
for (i = 1; i < size; i += 4)
|
||||
if (_uccomp_data[i] != 2)
|
||||
break;
|
||||
_uccomp_size = i - 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
_uccomp_unload(void)
|
||||
{
|
||||
if (_uccomp_size == 0)
|
||||
return;
|
||||
|
||||
free((char *) _uccomp_data);
|
||||
_uccomp_size = 0;
|
||||
}
|
||||
|
||||
int
|
||||
uccomp(unsigned long node1, unsigned long node2, unsigned long *comp)
|
||||
{
|
||||
int l, r, m;
|
||||
|
||||
l = 0;
|
||||
r = _uccomp_size - 1;
|
||||
|
||||
while (l <= r) {
|
||||
m = ((r + l) >> 1);
|
||||
m -= m & 3;
|
||||
if (node1 > _uccomp_data[m+2])
|
||||
l = m + 4;
|
||||
else if (node1 < _uccomp_data[m+2])
|
||||
r = m - 4;
|
||||
else if (node2 > _uccomp_data[m+3])
|
||||
l = m + 4;
|
||||
else if (node2 < _uccomp_data[m+3])
|
||||
r = m - 4;
|
||||
else {
|
||||
*comp = _uccomp_data[m];
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
uccomp_hangul(unsigned long *str, int len)
|
||||
{
|
||||
const int SBase = 0xAC00, LBase = 0x1100,
|
||||
VBase = 0x1161, TBase = 0x11A7,
|
||||
LCount = 19, VCount = 21, TCount = 28,
|
||||
NCount = VCount * TCount, /* 588 */
|
||||
SCount = LCount * NCount; /* 11172 */
|
||||
|
||||
int i, rlen;
|
||||
unsigned long ch, last, lindex, sindex;
|
||||
|
||||
last = str[0];
|
||||
rlen = 1;
|
||||
for ( i = 1; i < len; i++ ) {
|
||||
ch = str[i];
|
||||
|
||||
/* check if two current characters are L and V */
|
||||
lindex = last - LBase;
|
||||
if (0 <= lindex && lindex < LCount) {
|
||||
unsigned long vindex = ch - VBase;
|
||||
if (0 <= vindex && vindex < VCount) {
|
||||
/* make syllable of form LV */
|
||||
last = SBase + (lindex * VCount + vindex) * TCount;
|
||||
str[rlen-1] = last; /* reset last */
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* check if two current characters are LV and T */
|
||||
sindex = last - SBase;
|
||||
if (0 <= sindex && sindex < SCount && (sindex % TCount) == 0) {
|
||||
unsigned long tindex = ch - TBase;
|
||||
if (0 <= tindex && tindex <= TCount) {
|
||||
/* make syllable of form LVT */
|
||||
last += tindex;
|
||||
str[rlen-1] = last; /* reset last */
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* if neither case was true, just add the character */
|
||||
last = ch;
|
||||
str[rlen] = ch;
|
||||
rlen++;
|
||||
}
|
||||
return rlen;
|
||||
}
|
||||
|
||||
int
|
||||
uccanoncomp(unsigned long *str, int len)
|
||||
{
|
||||
int i, stpos, copos;
|
||||
unsigned long cl, prevcl, st, ch, co;
|
||||
|
||||
st = str[0];
|
||||
stpos = 0;
|
||||
copos = 1;
|
||||
prevcl = uccombining_class(st) == 0 ? 0 : 256;
|
||||
|
||||
for (i = 1; i < len; i++) {
|
||||
ch = str[i];
|
||||
cl = uccombining_class(ch);
|
||||
if (uccomp(st, ch, &co) && (prevcl < cl || prevcl == 0))
|
||||
st = str[stpos] = co;
|
||||
else {
|
||||
if (cl == 0) {
|
||||
stpos = copos;
|
||||
st = ch;
|
||||
}
|
||||
prevcl = cl;
|
||||
str[copos++] = ch;
|
||||
}
|
||||
}
|
||||
|
||||
return uccomp_hangul(str, copos);
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Support for decompositions.
|
||||
@ -491,9 +678,9 @@ _ucdcmp_load(char *paths, int reload)
|
||||
|
||||
if (_ucdcmp_size > 0) {
|
||||
if (!reload)
|
||||
/*
|
||||
* The decompositions have already been loaded.
|
||||
*/
|
||||
/*
|
||||
* The decompositions have already been loaded.
|
||||
*/
|
||||
return 0;
|
||||
|
||||
free((char *) _ucdcmp_nodes);
|
||||
@ -501,7 +688,7 @@ _ucdcmp_load(char *paths, int reload)
|
||||
}
|
||||
|
||||
if ((in = _ucopenfile(paths, "decomp.dat", "rb")) == 0)
|
||||
return -1;
|
||||
return -1;
|
||||
|
||||
/*
|
||||
* Load the header.
|
||||
@ -528,7 +715,7 @@ _ucdcmp_load(char *paths, int reload)
|
||||
*/
|
||||
if (hdr.bom == 0xfffe) {
|
||||
for (i = 0; i < size; i++)
|
||||
_ucdcmp_nodes[i] = endian_long(_ucdcmp_nodes[i]);
|
||||
_ucdcmp_nodes[i] = endian_long(_ucdcmp_nodes[i]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -590,6 +777,74 @@ ucdecomp_hangul(unsigned long code, unsigned long *num, unsigned long decomp[])
|
||||
return 1;
|
||||
}
|
||||
|
||||
int
|
||||
uccanondecomp(const unsigned long *in, int inlen,
|
||||
unsigned long **out, int *outlen)
|
||||
{
|
||||
int i, j, k, l, size;
|
||||
unsigned long num, class, *decomp, hangdecomp[3];
|
||||
|
||||
size = inlen;
|
||||
*out = (unsigned long *) malloc(size * sizeof(**out));
|
||||
if (*out == NULL)
|
||||
return *outlen = -1;
|
||||
|
||||
i = 0;
|
||||
for (j = 0; j < inlen; j++) {
|
||||
if (ucdecomp(in[j], &num, &decomp)) {
|
||||
if (size - i < num) {
|
||||
size = inlen + i - j + num - 1;
|
||||
*out = (unsigned long *) realloc(*out, size * sizeof(**out));
|
||||
if (*out == NULL)
|
||||
return *outlen = -1;
|
||||
}
|
||||
for (k = 0; k < num; k++) {
|
||||
class = uccombining_class(decomp[k]);
|
||||
if (class == 0) {
|
||||
(*out)[i] = decomp[k];
|
||||
} else {
|
||||
for (l = i; l > 0; l--)
|
||||
if (class >= uccombining_class((*out)[l-1]))
|
||||
break;
|
||||
memmove(*out + l + 1, *out + l, (i - l) * sizeof(**out));
|
||||
(*out)[l] = decomp[k];
|
||||
}
|
||||
i++;
|
||||
}
|
||||
} else if (ucdecomp_hangul(in[j], &num, hangdecomp)) {
|
||||
if (size - i < num) {
|
||||
size = inlen + i - j + num - 1;
|
||||
*out = (unsigned long *) realloc(*out, size * sizeof(**out));
|
||||
if (*out == NULL)
|
||||
return *outlen = -1;
|
||||
}
|
||||
for (k = 0; k < num; k++) {
|
||||
(*out)[i] = hangdecomp[k];
|
||||
i++;
|
||||
}
|
||||
} else {
|
||||
if (size - i < 1) {
|
||||
size = inlen + i - j;
|
||||
*out = (unsigned long *) realloc(*out, size * sizeof(**out));
|
||||
if (*out == NULL)
|
||||
return *outlen = -1;
|
||||
}
|
||||
class = uccombining_class(in[j]);
|
||||
if (class == 0) {
|
||||
(*out)[i] = in[j];
|
||||
} else {
|
||||
for (l = i; l > 0; l--)
|
||||
if (class >= uccombining_class((*out)[l-1]))
|
||||
break;
|
||||
memmove(*out + l + 1, *out + l, (i - l) * sizeof(**out));
|
||||
(*out)[l] = in[j];
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return *outlen = i;
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Support for combining classes.
|
||||
@ -611,17 +866,17 @@ _uccmcl_load(char *paths, int reload)
|
||||
|
||||
if (_uccmcl_size > 0) {
|
||||
if (!reload)
|
||||
/*
|
||||
* The combining classes have already been loaded.
|
||||
*/
|
||||
return 0;
|
||||
/*
|
||||
* The combining classes have already been loaded.
|
||||
*/
|
||||
return 0;
|
||||
|
||||
free((char *) _uccmcl_nodes);
|
||||
_uccmcl_size = 0;
|
||||
}
|
||||
|
||||
if ((in = _ucopenfile(paths, "cmbcl.dat", "rb")) == 0)
|
||||
return -1;
|
||||
return -1;
|
||||
|
||||
/*
|
||||
* Load the header.
|
||||
@ -646,7 +901,7 @@ _uccmcl_load(char *paths, int reload)
|
||||
*/
|
||||
if (hdr.bom == 0xfffe) {
|
||||
for (i = 0; i < _uccmcl_size; i++)
|
||||
_uccmcl_nodes[i] = endian_long(_uccmcl_nodes[i]);
|
||||
_uccmcl_nodes[i] = endian_long(_uccmcl_nodes[i]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -872,16 +1127,20 @@ int
|
||||
ucdata_load(char *paths, int masks)
|
||||
{
|
||||
int error = 0;
|
||||
|
||||
if (masks & UCDATA_CTYPE)
|
||||
error |= _ucprop_load(paths, 0) < 0 ? UCDATA_CTYPE : 0;
|
||||
error |= _ucprop_load(paths, 0) < 0 ? UCDATA_CTYPE : 0;
|
||||
if (masks & UCDATA_CASE)
|
||||
error |= _uccase_load(paths, 0) < 0 ? UCDATA_CASE : 0;
|
||||
error |= _uccase_load(paths, 0) < 0 ? UCDATA_CASE : 0;
|
||||
if (masks & UCDATA_DECOMP)
|
||||
error |= _ucdcmp_load(paths, 0) < 0 ? UCDATA_DECOMP : 0;
|
||||
error |= _ucdcmp_load(paths, 0) < 0 ? UCDATA_DECOMP : 0;
|
||||
if (masks & UCDATA_CMBCL)
|
||||
error |= _uccmcl_load(paths, 0) < 0 ? UCDATA_CMBCL : 0;
|
||||
error |= _uccmcl_load(paths, 0) < 0 ? UCDATA_CMBCL : 0;
|
||||
if (masks & UCDATA_NUM)
|
||||
error |= _ucnumb_load(paths, 0) < 0 ? UCDATA_NUM : 0;
|
||||
error |= _ucnumb_load(paths, 0) < 0 ? UCDATA_NUM : 0;
|
||||
if (masks & UCDATA_COMP)
|
||||
error |= _uccomp_load(paths, 0) < 0 ? UCDATA_COMP : 0;
|
||||
|
||||
return -error;
|
||||
}
|
||||
|
||||
@ -898,6 +1157,8 @@ ucdata_unload(int masks)
|
||||
_uccmcl_unload();
|
||||
if (masks & UCDATA_NUM)
|
||||
_ucnumb_unload();
|
||||
if (masks & UCDATA_COMP)
|
||||
_uccomp_unload();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -907,16 +1168,20 @@ int
|
||||
ucdata_reload(char *paths, int masks)
|
||||
{
|
||||
int error = 0;
|
||||
|
||||
if (masks & UCDATA_CTYPE)
|
||||
error |= _ucprop_load(paths, 1) < 0 ? UCDATA_CTYPE : 0;
|
||||
error |= _ucprop_load(paths, 1) < 0 ? UCDATA_CTYPE : 0;
|
||||
if (masks & UCDATA_CASE)
|
||||
error |= _uccase_load(paths, 1) < 0 ? UCDATA_CASE : 0;
|
||||
error |= _uccase_load(paths, 1) < 0 ? UCDATA_CASE : 0;
|
||||
if (masks & UCDATA_DECOMP)
|
||||
error |= _ucdcmp_load(paths, 1) < 0 ? UCDATA_DECOMP : 0;
|
||||
error |= _ucdcmp_load(paths, 1) < 0 ? UCDATA_DECOMP : 0;
|
||||
if (masks & UCDATA_CMBCL)
|
||||
error |= _uccmcl_load(paths, 1) < 0 ? UCDATA_CMBCL : 0;
|
||||
error |= _uccmcl_load(paths, 1) < 0 ? UCDATA_CMBCL : 0;
|
||||
if (masks & UCDATA_NUM)
|
||||
error |= _ucnumb_load(paths, 1) < 0 ? UCDATA_NUM : 0;
|
||||
error |= _ucnumb_load(paths, 1) < 0 ? UCDATA_NUM : 0;
|
||||
if (masks & UCDATA_COMP)
|
||||
error |= _uccomp_load(paths, 1) < 0 ? UCDATA_COMP : 0;
|
||||
|
||||
return -error;
|
||||
}
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
* COPYING RESTRICTIONS APPLY, see COPYRIGHT file
|
||||
*/
|
||||
/*
|
||||
* Copyright 1999 Computing Research Labs, New Mexico State University
|
||||
* Copyright 2001 Computing Research Labs, New Mexico State University
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
@ -28,7 +28,7 @@
|
||||
#define _h_ucdata
|
||||
|
||||
/*
|
||||
* $Id: ucdata.h,v 1.5 1999/11/19 15:24:29 mleisher Exp $
|
||||
* $Id: ucdata.h,v 1.6 2001/01/02 18:46:20 mleisher Exp $
|
||||
*/
|
||||
|
||||
LDAP_BEGIN_DECL
|
||||
@ -36,7 +36,7 @@ LDAP_BEGIN_DECL
|
||||
#undef __
|
||||
#define __(x) x
|
||||
|
||||
#define UCDATA_VERSION "2.3"
|
||||
#define UCDATA_VERSION "2.4"
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
@ -210,6 +210,32 @@ extern unsigned long uctoupper __((unsigned long code));
|
||||
extern unsigned long uctolower __((unsigned long code));
|
||||
extern unsigned long uctotitle __((unsigned long code));
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Functions for getting compositions.
|
||||
*
|
||||
**************************************************************************/
|
||||
|
||||
/*
|
||||
* This routine determines if there exists a composition of node1 and node2.
|
||||
* If it returns 0, there is no composition. Any other value indicates a
|
||||
* composition was returned in comp.
|
||||
*/
|
||||
extern int uccomp __((unsigned long node1, unsigned long node2,
|
||||
unsigned long *comp));
|
||||
|
||||
/*
|
||||
* Does Hangul composition on the string str with length len, and returns
|
||||
* the length of the composed string.
|
||||
*/
|
||||
extern int uccomp_hangul __((unsigned long *str, int len));
|
||||
|
||||
/*
|
||||
* Does canonical composition on the string str with length len, and returns
|
||||
* the length of the composed string.
|
||||
*/
|
||||
extern int uccanoncomp __((unsigned long *str, int len));
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Functions for getting decompositions.
|
||||
@ -222,7 +248,6 @@ extern unsigned long uctotitle __((unsigned long code));
|
||||
* returned.
|
||||
*/
|
||||
extern int ucdecomp __((unsigned long code, unsigned long *num,
|
||||
|
||||
unsigned long **decomp));
|
||||
|
||||
/*
|
||||
@ -232,6 +257,15 @@ extern int ucdecomp __((unsigned long code, unsigned long *num,
|
||||
extern int ucdecomp_hangul __((unsigned long code, unsigned long *num,
|
||||
unsigned long decomp[]));
|
||||
|
||||
/*
|
||||
* This routine does canonical decomposition of the string in of length
|
||||
* inlen, and returns the decomposed string in out with length outlen.
|
||||
* The memory for out is allocated by this routine. It returns the length
|
||||
* of the decomposed string if okay, and -1 on error.
|
||||
*/
|
||||
extern int uccanondecomp __((const unsigned long *in, int inlen,
|
||||
unsigned long **out, int *outlen));
|
||||
|
||||
/**************************************************************************
|
||||
*
|
||||
* Functions for getting combining classes.
|
||||
@ -279,9 +313,10 @@ extern int ucgetdigit __((unsigned long code));
|
||||
#define UCDATA_DECOMP 0x04
|
||||
#define UCDATA_CMBCL 0x08
|
||||
#define UCDATA_NUM 0x10
|
||||
#define UCDATA_COMP 0x20
|
||||
|
||||
#define UCDATA_ALL (UCDATA_CASE|UCDATA_CTYPE|UCDATA_DECOMP|\
|
||||
UCDATA_CMBCL|UCDATA_NUM)
|
||||
UCDATA_CMBCL|UCDATA_NUM|UCDATA_COMP)
|
||||
|
||||
/*
|
||||
* Functions to load, unload, and reload specific data files.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/* $OpenLDAP$ */
|
||||
/*
|
||||
* Copyright 1999 Computing Research Labs, New Mexico State University
|
||||
* Copyright 2001 Computing Research Labs, New Mexico State University
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
@ -20,7 +20,7 @@
|
||||
* OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
|
||||
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
/* $Id: ucgendat.c,v 1.3 1999/10/07 20:49:56 mleisher Exp $" */
|
||||
/* $Id: ucgendat.c,v 1.4 2001/01/02 18:46:20 mleisher Exp $" */
|
||||
|
||||
#include "portable.h"
|
||||
|
||||
@ -39,7 +39,7 @@
|
||||
*/
|
||||
static unsigned short hdr[2] = {0xfeff, 0};
|
||||
|
||||
#define NUMPROPS 49
|
||||
#define NUMPROPS 50
|
||||
#define NEEDPROPS (NUMPROPS + (4 - (NUMPROPS & 3)))
|
||||
|
||||
typedef struct {
|
||||
@ -70,7 +70,7 @@ static _prop_t props[NUMPROPS] = {
|
||||
{"EN", 2}, {"ES", 2}, {"ET", 2}, {"AN", 2}, {"CS", 2}, {"B", 1},
|
||||
{"S", 1}, {"WS", 2}, {"ON", 2},
|
||||
{"Cm", 2}, {"Nb", 2}, {"Sy", 2}, {"Hd", 2}, {"Qm", 2}, {"Mr", 2},
|
||||
{"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}
|
||||
{"Ss", 2}, {"Cp", 2}, {"Pi", 2}, {"Pf", 2}, {"AL", 2}
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
@ -108,6 +108,26 @@ static _decomp_t *decomps;
|
||||
static unsigned long decomps_used;
|
||||
static unsigned long decomps_size;
|
||||
|
||||
/*
|
||||
* Composition exclusion table stuff.
|
||||
*/
|
||||
#define COMPEX_SET(c) (compexs[(c) >> 15] |= (1 << ((c) & 31)))
|
||||
#define COMPEX_TEST(c) (compexs[(c) >> 15] & (1 << ((c) & 31)))
|
||||
static unsigned long compexs[2048];
|
||||
|
||||
/*
|
||||
* Struct for holding a composition pair, and array of composition pairs
|
||||
*/
|
||||
typedef struct {
|
||||
unsigned long comp;
|
||||
unsigned long count;
|
||||
unsigned long code1;
|
||||
unsigned long code2;
|
||||
} _comp_t;
|
||||
|
||||
static _comp_t *comps;
|
||||
static unsigned long comps_used;
|
||||
|
||||
/*
|
||||
* Types and lists for handling lists of case mappings.
|
||||
*/
|
||||
@ -281,24 +301,11 @@ ordered_range_insert(unsigned long c, char *name, int len)
|
||||
/*
|
||||
* Deal with directionality codes introduced in Unicode 3.0.
|
||||
*/
|
||||
if (len == 2) {
|
||||
if (memcmp(name, "AL", 2) == 0) {
|
||||
/*
|
||||
* Mark the Arabic letters as having RTL directionality.
|
||||
*/
|
||||
len = 1;
|
||||
name = "R";
|
||||
} else if (memcmp(name, "BN", 2) == 0) {
|
||||
/*
|
||||
* Mark the control characters as being Other Neutrals.
|
||||
*/
|
||||
len = 2;
|
||||
name = "ON";
|
||||
}
|
||||
} else if (len == 3 &&
|
||||
(memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
|
||||
memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
|
||||
memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0)) {
|
||||
if ((len == 2 && memcmp(name, "BN", 2) == 0) ||
|
||||
(len == 3 &&
|
||||
(memcmp(name, "NSM", 3) == 0 || memcmp(name, "PDF", 3) == 0 ||
|
||||
memcmp(name, "LRE", 3) == 0 || memcmp(name, "LRO", 3) == 0 ||
|
||||
memcmp(name, "RLE", 3) == 0 || memcmp(name, "RLO", 3) == 0))) {
|
||||
/*
|
||||
* Mark all of these as Other Neutral to preserve compatibility with
|
||||
* older versions.
|
||||
@ -468,6 +475,12 @@ add_decomp(unsigned long code)
|
||||
(void) AC_MEMCPY((char *) decomps[i].decomp, (char *) dectmp,
|
||||
sizeof(unsigned long) * dectmp_size);
|
||||
|
||||
/*
|
||||
* NOTICE: This needs changing later so it is more general than simply
|
||||
* pairs. This calculation is done here to simplify allocation elsewhere.
|
||||
*/
|
||||
if (dectmp_size == 2)
|
||||
comps_used++;
|
||||
}
|
||||
|
||||
static void
|
||||
@ -938,10 +951,10 @@ read_cdata(FILE *in)
|
||||
}
|
||||
|
||||
/*
|
||||
* If there is more than one code in the temporary decomposition
|
||||
* array, then add the character with its decomposition.
|
||||
* If there are any codes in the temporary decomposition array,
|
||||
* then add the character with its decomposition.
|
||||
*/
|
||||
if (dectmp_size > 1)
|
||||
if (dectmp_size > 0)
|
||||
add_decomp(code);
|
||||
}
|
||||
|
||||
@ -1082,6 +1095,74 @@ expand_decomp(void)
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
cmpcomps(_comp_t *comp1, _comp_t *comp2)
|
||||
{
|
||||
long diff = comp1->code1 - comp2->code1;
|
||||
|
||||
if (!diff)
|
||||
diff = comp1->code2 - comp2->code2;
|
||||
return (int) diff;
|
||||
}
|
||||
|
||||
/*
|
||||
* Load composition exclusion data
|
||||
*/
|
||||
static void
|
||||
read_compexdata(FILE *in)
|
||||
{
|
||||
unsigned short i, code;
|
||||
char line[512], *s;
|
||||
|
||||
(void) memset((char *) compexs, 0, sizeof(unsigned long) << 11);
|
||||
|
||||
while (fscanf(in, "%[^\n]\n", line) != EOF) {
|
||||
/*
|
||||
* Skip blank lines and lines that start with a '#'.
|
||||
*/
|
||||
if (line[0] == 0 || line[0] == '#')
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Collect the code. Assume max 4 digits
|
||||
*/
|
||||
|
||||
for (s = line, i = code = 0; *s != '#' && i < 4; i++, s++) {
|
||||
code <<= 4;
|
||||
if (*s >= '0' && *s <= '9')
|
||||
code += *s - '0';
|
||||
else if (*s >= 'A' && *s <= 'F')
|
||||
code += (*s - 'A') + 10;
|
||||
else if (*s >= 'a' && *s <= 'f')
|
||||
code += (*s - 'a') + 10;
|
||||
}
|
||||
COMPEX_SET(code);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates array of compositions from decomposition array
|
||||
*/
|
||||
static void
|
||||
create_comps(void)
|
||||
{
|
||||
unsigned long i, cu;
|
||||
|
||||
comps = (_comp_t *) malloc(comps_used * sizeof(_comp_t));
|
||||
|
||||
for (i = cu = 0; i < decomps_used; i++) {
|
||||
if (decomps[i].used != 2 || COMPEX_TEST(decomps[i].code))
|
||||
continue;
|
||||
comps[cu].comp = decomps[i].code;
|
||||
comps[cu].count = 2;
|
||||
comps[cu].code1 = decomps[i].decomp[0];
|
||||
comps[cu].code2 = decomps[i].decomp[1];
|
||||
cu++;
|
||||
}
|
||||
qsort(comps, comps_used, sizeof(_comp_t),
|
||||
(int (*)(const void *, const void *)) cmpcomps);
|
||||
}
|
||||
|
||||
static void
|
||||
write_cdata(char *opath)
|
||||
{
|
||||
@ -1211,6 +1292,44 @@ write_cdata(char *opath)
|
||||
|
||||
fclose(out);
|
||||
|
||||
/*****************************************************************
|
||||
*
|
||||
* Generate the composition data.
|
||||
*
|
||||
*****************************************************************/
|
||||
|
||||
/*
|
||||
* Create compositions from decomposition data
|
||||
*/
|
||||
create_comps();
|
||||
|
||||
/*
|
||||
* Open the comp.dat file.
|
||||
*/
|
||||
sprintf(path, "%s/comp.dat", opath);
|
||||
if ((out = fopen(path, "wb")) == 0)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Write the header.
|
||||
*/
|
||||
hdr[1] = (unsigned short) comps_used * 4;
|
||||
fwrite((char *) hdr, sizeof(unsigned short), 2, out);
|
||||
|
||||
/*
|
||||
* Write out the byte count to maintain header size.
|
||||
*/
|
||||
bytes = comps_used * sizeof(_comp_t);
|
||||
fwrite((char *) &bytes, sizeof(unsigned long), 1, out);
|
||||
|
||||
/*
|
||||
* Now, if comps exist, write them out.
|
||||
*/
|
||||
if (comps_used > 0)
|
||||
fwrite((char *) comps, sizeof(_comp_t), comps_used, out);
|
||||
|
||||
fclose(out);
|
||||
|
||||
/*****************************************************************
|
||||
*
|
||||
* Generate the decomposition data.
|
||||
@ -1355,6 +1474,21 @@ write_cdata(char *opath)
|
||||
fclose(out);
|
||||
}
|
||||
|
||||
static void
|
||||
usage(char *prog)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Usage: %s [-o output-directory|-x composition-exclusions]", prog);
|
||||
fprintf(stderr, " datafile1 datafile2 ...\n\n");
|
||||
fprintf(stderr,
|
||||
"-o output-directory\n\t\tWrite the output files to a different");
|
||||
fprintf(stderr, " directory (default: .).\n");
|
||||
fprintf(stderr,
|
||||
"-x composition-exclusion\n\t\tFile of composition codes");
|
||||
fprintf(stderr, " that should be excluded.\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char *argv[])
|
||||
{
|
||||
@ -1373,10 +1507,29 @@ main(int argc, char *argv[])
|
||||
argv++;
|
||||
|
||||
while (argc > 0) {
|
||||
if (argv[0][0] == '-' && argv[0][1] == 'o') {
|
||||
argc--;
|
||||
argv++;
|
||||
opath = argv[0];
|
||||
if (argv[0][0] == '-') {
|
||||
switch (argv[0][1]) {
|
||||
case 'o':
|
||||
argc--;
|
||||
argv++;
|
||||
opath = argv[0];
|
||||
break;
|
||||
case 'x':
|
||||
argc--;
|
||||
argv++;
|
||||
if ((in = fopen(argv[0], "rb")) == 0)
|
||||
fprintf(stderr,
|
||||
"%s: unable to open composition exclusion file %s\n",
|
||||
prog, argv[0]);
|
||||
else {
|
||||
read_compexdata(in);
|
||||
fclose(in);
|
||||
in = 0;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
usage(prog);
|
||||
}
|
||||
} else {
|
||||
if (in != stdin && in != NULL)
|
||||
fclose(in);
|
||||
@ -1387,7 +1540,7 @@ main(int argc, char *argv[])
|
||||
read_cdata(in);
|
||||
fclose(in);
|
||||
in = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
argc--;
|
||||
argv++;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user