postgresql/contrib/unaccent/unaccent.c

321 lines
6.7 KiB
C
Raw Normal View History

2009-08-18 18:34:39 +08:00
/*-------------------------------------------------------------------------
*
* unaccent.c
2010-02-26 10:01:40 +08:00
* Text search unaccent dictionary
2009-08-18 18:34:39 +08:00
*
2010-01-03 00:58:17 +08:00
* Copyright (c) 2009-2010, PostgreSQL Global Development Group
2009-08-18 18:34:39 +08:00
*
* IDENTIFICATION
* $PostgreSQL: pgsql/contrib/unaccent/unaccent.c,v 1.6 2010/08/05 15:25:35 rhaas Exp $
2009-08-18 18:34:39 +08:00
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "fmgr.h"
#include "catalog/namespace.h"
#include "commands/defrem.h"
#include "mb/pg_wchar.h"
#include "tsearch/ts_cache.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_public.h"
#include "utils/builtins.h"
PG_MODULE_MAGIC;
/*
2010-02-26 10:01:40 +08:00
* Unaccent dictionary uses uncompressed suffix tree to find a
* character to replace. Each node of tree is an array of
2009-08-18 18:34:39 +08:00
* SuffixChar struct with length = 256 (n-th element of array
* corresponds to byte)
*/
2010-02-26 10:01:40 +08:00
typedef struct SuffixChar
{
struct SuffixChar *nextChar;
char *replaceTo;
int replacelen;
2009-08-18 18:34:39 +08:00
} SuffixChar;
/*
* placeChar - put str into tree's structure, byte by byte.
*/
2010-02-26 10:01:40 +08:00
static SuffixChar *
2009-08-18 18:34:39 +08:00
placeChar(SuffixChar *node, unsigned char *str, int lenstr, char *replaceTo, int replacelen)
{
2010-02-26 10:01:40 +08:00
SuffixChar *curnode;
2009-08-18 18:34:39 +08:00
2010-02-26 10:01:40 +08:00
if (!node)
2009-08-18 18:34:39 +08:00
{
node = palloc(sizeof(SuffixChar) * 256);
memset(node, 0, sizeof(SuffixChar) * 256);
}
curnode = node + *str;
2010-02-26 10:01:40 +08:00
if (lenstr == 1)
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
if (curnode->replaceTo)
2009-08-18 18:34:39 +08:00
elog(WARNING, "duplicate TO argument, use first one");
else
{
curnode->replacelen = replacelen;
2010-02-26 10:01:40 +08:00
curnode->replaceTo = palloc(replacelen);
2009-08-18 18:34:39 +08:00
memcpy(curnode->replaceTo, replaceTo, replacelen);
}
}
else
{
2010-02-26 10:01:40 +08:00
curnode->nextChar = placeChar(curnode->nextChar, str + 1, lenstr - 1, replaceTo, replacelen);
2009-08-18 18:34:39 +08:00
}
return node;
}
/*
* initSuffixTree - create suffix tree from file. Function converts
* UTF8-encoded file into current encoding.
*/
2010-02-26 10:01:40 +08:00
static SuffixChar *
initSuffixTree(char *filename)
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
SuffixChar *volatile rootSuffixTree = NULL;
2009-08-18 18:34:39 +08:00
MemoryContext ccxt = CurrentMemoryContext;
2010-02-26 10:01:40 +08:00
tsearch_readline_state trst;
volatile bool skip;
2009-08-18 18:34:39 +08:00
filename = get_tsearch_config_filename(filename, "rules");
if (!tsearch_readline_begin(&trst, filename))
ereport(ERROR,
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("could not open unaccent file \"%s\": %m",
filename)));
2010-02-26 10:01:40 +08:00
do
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
char src[4096];
char trg[4096];
int srclen;
int trglen;
char *line = NULL;
2009-08-18 18:34:39 +08:00
skip = true;
PG_TRY();
{
/*
2010-02-26 10:01:40 +08:00
* pg_do_encoding_conversion() (called by tsearch_readline()) will
* emit exception if it finds untranslatable characters in current
* locale. We just skip such characters.
2009-08-18 18:34:39 +08:00
*/
while ((line = tsearch_readline(&trst)) != NULL)
{
2010-02-26 10:01:40 +08:00
if (sscanf(line, "%s\t%s\n", src, trg) != 2)
2009-08-18 18:34:39 +08:00
continue;
srclen = strlen(src);
trglen = strlen(trg);
2010-02-26 10:01:40 +08:00
rootSuffixTree = placeChar(rootSuffixTree,
(unsigned char *) src, srclen,
trg, trglen);
2009-08-18 18:34:39 +08:00
skip = false;
pfree(line);
}
}
PG_CATCH();
{
ErrorData *errdata;
MemoryContext ecxt;
ecxt = MemoryContextSwitchTo(ccxt);
errdata = CopyErrorData();
if (errdata->sqlerrcode == ERRCODE_UNTRANSLATABLE_CHARACTER)
{
FlushErrorState();
}
else
{
MemoryContextSwitchTo(ecxt);
PG_RE_THROW();
}
}
PG_END_TRY();
}
2010-02-26 10:01:40 +08:00
while (skip);
2009-08-18 18:34:39 +08:00
tsearch_readline_end(&trst);
return rootSuffixTree;
}
/*
* findReplaceTo - find multibyte character in tree
*/
2010-02-26 10:01:40 +08:00
static SuffixChar *
findReplaceTo(SuffixChar *node, unsigned char *src, int srclen)
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
while (node)
2009-08-18 18:34:39 +08:00
{
node = node + *src;
2010-02-26 10:01:40 +08:00
if (srclen == 1)
2009-08-18 18:34:39 +08:00
return node;
src++;
srclen--;
node = node->nextChar;
}
return NULL;
}
PG_FUNCTION_INFO_V1(unaccent_init);
2010-02-26 10:01:40 +08:00
Datum unaccent_init(PG_FUNCTION_ARGS);
2009-08-18 18:34:39 +08:00
Datum
unaccent_init(PG_FUNCTION_ARGS)
{
2010-02-26 10:01:40 +08:00
List *dictoptions = (List *) PG_GETARG_POINTER(0);
SuffixChar *rootSuffixTree = NULL;
2010-02-26 10:01:40 +08:00
bool fileloaded = false;
2009-08-18 18:34:39 +08:00
ListCell *l;
foreach(l, dictoptions)
{
DefElem *defel = (DefElem *) lfirst(l);
if (pg_strcasecmp("Rules", defel->defname) == 0)
{
if (fileloaded)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("multiple Rules parameters")));
2010-02-26 10:01:40 +08:00
rootSuffixTree = initSuffixTree(defGetString(defel));
fileloaded = true;
2009-08-18 18:34:39 +08:00
}
else
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("unrecognized Unaccent parameter: \"%s\"",
defel->defname)));
}
}
if (!fileloaded)
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("missing Rules parameter")));
}
PG_RETURN_POINTER(rootSuffixTree);
}
PG_FUNCTION_INFO_V1(unaccent_lexize);
2010-02-26 10:01:40 +08:00
Datum unaccent_lexize(PG_FUNCTION_ARGS);
2009-08-18 18:34:39 +08:00
Datum
unaccent_lexize(PG_FUNCTION_ARGS)
{
2010-02-26 10:01:40 +08:00
SuffixChar *rootSuffixTree = (SuffixChar *) PG_GETARG_POINTER(0);
char *srcchar = (char *) PG_GETARG_POINTER(1);
2009-08-18 18:34:39 +08:00
int32 len = PG_GETARG_INT32(2);
2010-02-26 10:01:40 +08:00
char *srcstart,
*trgchar = NULL;
2009-08-18 18:34:39 +08:00
int charlen;
TSLexeme *res = NULL;
SuffixChar *node;
srcstart = srcchar;
2010-02-26 10:01:40 +08:00
while (srcchar - srcstart < len)
2009-08-18 18:34:39 +08:00
{
charlen = pg_mblen(srcchar);
2010-02-26 10:01:40 +08:00
node = findReplaceTo(rootSuffixTree, (unsigned char *) srcchar, charlen);
if (node && node->replaceTo)
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
if (!res)
2009-08-18 18:34:39 +08:00
{
/* allocate res only it it's needed */
res = palloc0(sizeof(TSLexeme) * 2);
2010-02-26 10:01:40 +08:00
res->lexeme = trgchar = palloc(len * pg_database_encoding_max_length() + 1 /* \0 */ );
2009-08-18 18:34:39 +08:00
res->flags = TSL_FILTER;
2010-02-26 10:01:40 +08:00
if (srcchar != srcstart)
2009-08-18 18:34:39 +08:00
{
memcpy(trgchar, srcstart, srcchar - srcstart);
trgchar += (srcchar - srcstart);
}
}
2010-02-26 10:01:40 +08:00
memcpy(trgchar, node->replaceTo, node->replacelen);
trgchar += node->replacelen;
2009-08-18 18:34:39 +08:00
}
2010-02-26 10:01:40 +08:00
else if (res)
2009-08-18 18:34:39 +08:00
{
2010-02-26 10:01:40 +08:00
memcpy(trgchar, srcchar, charlen);
2009-08-18 18:34:39 +08:00
trgchar += charlen;
}
srcchar += charlen;
}
2010-02-26 10:01:40 +08:00
if (res)
2009-08-18 18:34:39 +08:00
*trgchar = '\0';
PG_RETURN_POINTER(res);
}
/*
* Function-like wrapper for dictionary
*/
PG_FUNCTION_INFO_V1(unaccent_dict);
2010-02-26 10:01:40 +08:00
Datum unaccent_dict(PG_FUNCTION_ARGS);
2009-08-18 18:34:39 +08:00
Datum
unaccent_dict(PG_FUNCTION_ARGS)
{
2010-02-26 10:01:40 +08:00
text *str;
int strArg;
Oid dictOid;
TSDictionaryCacheEntry *dict;
TSLexeme *res;
2009-08-18 18:34:39 +08:00
if (PG_NARGS() == 1)
{
dictOid = get_ts_dict_oid(stringToQualifiedNameList("unaccent"), false);
2009-08-18 18:34:39 +08:00
strArg = 0;
}
else
{
dictOid = PG_GETARG_OID(0);
strArg = 1;
}
str = PG_GETARG_TEXT_P(strArg);
dict = lookup_ts_dictionary_cache(dictOid);
res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
2010-02-26 10:01:40 +08:00
PointerGetDatum(dict->dictData),
PointerGetDatum(VARDATA(str)),
Int32GetDatum(VARSIZE(str) - VARHDRSZ),
2009-08-18 18:34:39 +08:00
PointerGetDatum(NULL)));
PG_FREE_IF_COPY(str, strArg);
2010-02-26 10:01:40 +08:00
if (res == NULL)
2009-08-18 18:34:39 +08:00
{
PG_RETURN_TEXT_P(PG_GETARG_TEXT_P_COPY(strArg));
}
2010-02-26 10:01:40 +08:00
else if (res->lexeme == NULL)
2009-08-18 18:34:39 +08:00
{
pfree(res);
PG_RETURN_TEXT_P(PG_GETARG_TEXT_P_COPY(strArg));
}
else
{
2010-02-26 10:01:40 +08:00
text *txt = cstring_to_text(res->lexeme);
2009-08-18 18:34:39 +08:00
pfree(res->lexeme);
pfree(res);
PG_RETURN_TEXT_P(txt);
}
}