postgresql/contrib/tsearch/gistidx.c

#include "postgres.h"

#include <float.h>

#include "access/gist.h"
#include "access/itup.h"
#include "access/rtree.h"
#include "utils/elog.h"
#include "utils/palloc.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "storage/bufpage.h"
#include "access/tuptoaster.h"

#include "txtidx.h"
#include "query.h"
#include "gistidx.h"
#include "crc32.h"

PG_FUNCTION_INFO_V1( gtxtidx_in );
Datum   gtxtidx_in(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_out );
Datum   gtxtidx_out(PG_FUNCTION_ARGS);

PG_FUNCTION_INFO_V1( gtxtidx_compress );
Datum   gtxtidx_compress(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_decompress );
Datum   gtxtidx_decompress(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_consistent );
Datum   gtxtidx_consistent(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_union );
Datum   gtxtidx_union(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_same );
Datum   gtxtidx_same(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_penalty );
Datum   gtxtidx_penalty(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1( gtxtidx_picksplit );
Datum   gtxtidx_picksplit(PG_FUNCTION_ARGS);

#define GETENTRY(vec,pos) ((GISTTYPE *) DatumGetPointer(((GISTENTRY *) VARDATA(vec))[(pos)].key))
#define SUMBIT(val) (        \
	GETBITBYTE(val,0) + \
	GETBITBYTE(val,1) + \
	GETBITBYTE(val,2) + \
	GETBITBYTE(val,3) + \
	GETBITBYTE(val,4) + \
	GETBITBYTE(val,5) + \
	GETBITBYTE(val,6) + \
	GETBITBYTE(val,7)   \
) 


Datum
gtxtidx_in(PG_FUNCTION_ARGS) {
	elog(ERROR,"Not implemented");
	PG_RETURN_DATUM(0);
}

Datum
gtxtidx_out(PG_FUNCTION_ARGS) {
	elog(ERROR,"Not implemented");
	PG_RETURN_DATUM(0);
}

static int
compareint( const void * a, const void * b ) {
	if ( *((int4*)a) == *((int4*)b) ) return 0;
	return ( *((int4*)a) > *((int4*)b) ) ? 1 : -1;
}

static int
uniqueint( int4* a, int4 l ) {
	int4 *ptr, *res;

	if ( l == 1 )
		return l;

	ptr = res = a;

	qsort((void*)a, l, sizeof(int4), compareint );

	while (ptr - a < l)
		if (*ptr != *res)
			*(++res) = *ptr++;
		else
			ptr++;
	return res + 1 - a;
}

static void
makesign( BITVECP sign, GISTTYPE *a) {
	int4 k,len = ARRNELEM( a );
	int4 *ptr = GETARR( a );
	MemSet( (void*)sign, 0, sizeof(BITVEC) );
	for(k=0;k<len;k++)
		HASH( sign, ptr[k] );
}

Datum
gtxtidx_compress(PG_FUNCTION_ARGS) {
	GISTENTRY *entry = (GISTENTRY *)PG_GETARG_POINTER(0);
	GISTENTRY  *retval = entry;

	if ( entry->leafkey ) { /* txtidx */
		GISTTYPE *res;
		txtidx	*toastedval = (txtidx*)DatumGetPointer( entry->key );
		txtidx	*val = (txtidx*)DatumGetPointer( PG_DETOAST_DATUM(entry->key) );
		int4 len;
		int4 *arr;
		WordEntry *ptr = ARRPTR(val);
		char *words = STRPTR(val);
		
		len = CALCGTSIZE( ARRKEY, val->size );
		res   = (GISTTYPE*)palloc( len );
		res->len = len;
		res->flag = ARRKEY;
		arr = GETARR(res);
		len = val->size;
		while( len-- ) {
			*arr = crc32_sz( (uint8*)&words[ ptr->pos ], ptr->len );
			arr++; ptr++; 
		}

		len = uniqueint( GETARR(res), val->size );
		if ( len != val->size ) { 
			/* there is a collision of hash-function;
			   len is always less than val->size */
			len = CALCGTSIZE( ARRKEY, len );
			res = (GISTTYPE*)repalloc( (void*)res, len );
			res->len = len;
		}
		if ( val != toastedval )
			pfree(val);

		/* make signature, if array is too long */
		if ( res->len > TOAST_INDEX_TARGET ) {
			GISTTYPE *ressign;

			len = CALCGTSIZE( SIGNKEY, 0 );
			ressign = (GISTTYPE*)palloc( len );
			ressign->len = len;
			ressign->flag = SIGNKEY;
			makesign( GETSIGN(ressign), res );
			pfree(res);
			res = ressign;
		}
		
		retval = (GISTENTRY*)palloc(sizeof(GISTENTRY));
		gistentryinit(*retval, PointerGetDatum(res), 
				entry->rel, entry->page,
				entry->offset, res->len, FALSE);
	} else if ( ISSIGNKEY(DatumGetPointer( entry->key )) && 
			! ISALLTRUE(DatumGetPointer( entry->key )) ){ 
		int4 i,len;
		GISTTYPE *res;
		BITVECP sign = GETSIGN( DatumGetPointer( entry->key ) );

		LOOPBYTE(
			if ( (sign[i] & 0xff) != 0xff )
				PG_RETURN_POINTER(retval);
		);

		len = CALCGTSIZE( SIGNKEY|ALLISTRUE, 0 );
		res    = (GISTTYPE*)palloc( len );
		res->len = len;
		res->flag = SIGNKEY | ALLISTRUE; 
		
		retval = (GISTENTRY*)palloc(sizeof(GISTENTRY));
		gistentryinit(*retval, PointerGetDatum(res), 
				entry->rel, entry->page,
				entry->offset, res->len, FALSE);
	}
	PG_RETURN_POINTER(retval);
}

Datum
gtxtidx_decompress(PG_FUNCTION_ARGS) {
	GISTENTRY *entry = (GISTENTRY *)PG_GETARG_POINTER(0);
	GISTTYPE  *key = (GISTTYPE*)DatumGetPointer( PG_DETOAST_DATUM(entry->key) );

	if ( key != (GISTTYPE*)DatumGetPointer(entry->key) ) {
		GISTENTRY *retval = (GISTENTRY*)palloc(sizeof(GISTENTRY));
		gistentryinit(*retval, PointerGetDatum(key), 
				entry->rel, entry->page,
				entry->offset, key->len, FALSE);

		PG_RETURN_POINTER(retval);
	}

	PG_RETURN_POINTER(entry);	
}

typedef struct {
	int4 *arrb;
	int4 *arre;
} CHKVAL;
 
/*
 * is there value 'val' in array or not ?
 */
static bool
checkcondition_arr( void *checkval, ITEM* val ) {
	int4      *StopLow = ((CHKVAL*)checkval)->arrb;
	int4      *StopHigh = ((CHKVAL*)checkval)->arre;
	int4      *StopMiddle;

	/* Loop invariant: StopLow <= val < StopHigh */
  
	while (StopLow < StopHigh) {
		StopMiddle = StopLow + (StopHigh - StopLow) / 2;
		if (*StopMiddle == val->val)
			return (true);
		else if (*StopMiddle < val->val )
			StopLow = StopMiddle + 1;
		else
			StopHigh = StopMiddle;
	}

	return (false);
}

static bool
checkcondition_bit( void *checkval, ITEM* val ) {
        return GETBIT( checkval, HASHVAL( val->val ) );
}

Datum 
gtxtidx_consistent(PG_FUNCTION_ARGS) {
	QUERYTYPE *query = (QUERYTYPE *)PG_GETARG_POINTER(1);
	GISTTYPE  *key   = (GISTTYPE  *)DatumGetPointer(
		((GISTENTRY *)PG_GETARG_POINTER(0))->key
	);

	if ( ISSIGNKEY(key) ) {
		if ( ISALLTRUE(key) )
			PG_RETURN_BOOL(true);

        	PG_RETURN_BOOL( execute(
                	GETQUERY(query),
                	(void*)GETSIGN(key), false,
                	checkcondition_bit
		));
	} else { /* only leaf pages */
		CHKVAL  chkval;

		chkval.arrb = GETARR(key);
        	chkval.arre = chkval.arrb + ARRNELEM(key);
        	PG_RETURN_BOOL( execute(
                	GETQUERY(query),
                	(void*)&chkval, true,
                	checkcondition_arr
        	) );
	}
}

static int4
unionkey( BITVECP sbase, GISTTYPE *add ) {
	int4	i;

	if ( ISSIGNKEY(add) ) {
		BITVECP sadd = GETSIGN( add );
		
		if ( ISALLTRUE(add) )
			return 1;

		LOOPBYTE(
			sbase[i] |= sadd[i];
		);
	} else {
		int4 *ptr = GETARR( add );
		for(i=0;i<ARRNELEM(add);i++)
			HASH( sbase, ptr[i] );
	}
	return 0;
}


Datum
gtxtidx_union(PG_FUNCTION_ARGS) {
	bytea *entryvec = (bytea *) PG_GETARG_POINTER(0);
	int *size = (int *) PG_GETARG_POINTER(1);
	BITVEC base;
	int4 len = (VARSIZE(entryvec) - VARHDRSZ) / sizeof(GISTENTRY);
	int4 i;
	int4 flag = 0;
	GISTTYPE *result;

	MemSet( (void*)base, 0, sizeof(BITVEC) );
	for(i=0;i<len;i++) {
		if ( unionkey( base, GETENTRY(entryvec, i) ) ) {
			flag = ALLISTRUE;
			break;
		}
	}

	flag |= SIGNKEY;
	len = CALCGTSIZE( flag, 0 );
	result = (GISTTYPE*)palloc( len );
	*size = result->len = len;
	result->flag = flag;
	if ( ! ISALLTRUE(result) )
		memcpy((void*)GETSIGN(result), (void*)base, sizeof( BITVEC ) );

	PG_RETURN_POINTER( result );
}

Datum
gtxtidx_same(PG_FUNCTION_ARGS) {
	GISTTYPE *a = (GISTTYPE*)PG_GETARG_POINTER(0);
	GISTTYPE *b = (GISTTYPE*)PG_GETARG_POINTER(1);
	bool *result = (bool *)PG_GETARG_POINTER(2);
	
	if ( ISSIGNKEY(a) ) { /* then b also ISSIGNKEY */
		if ( ISALLTRUE(a) && ISALLTRUE(b) ) {
			*result = true;
		} else if ( ISALLTRUE(a) ) {
			*result = false;
		} else if ( ISALLTRUE(b) ) {
			*result = false;
		} else {
			int4 i;
			BITVECP sa=GETSIGN(a), sb=GETSIGN(b);
			
			*result = true;
			LOOPBYTE(
				if ( sa[i] != sb[i] ) {
					*result = false;
					break;
				}
			);
		} 
	} else { /* a and b ISARRKEY */
		int4 lena = ARRNELEM(a), lenb = ARRNELEM(b);

		if ( lena != lenb ) {
			*result = false;
		} else {
			int4 *ptra = GETARR(a), *ptrb = GETARR(b);
			int4 i;

			*result = true;
			for(i=0;i<lena;i++)
				if ( ptra[i] != ptrb[i] ) {
					*result = false;
					break;
				}
		}
	}
	
	PG_RETURN_POINTER(result);
}

static int4
sizebitvec( BITVECP sign ) {
	int4 size=0, i;
	LOOPBYTE(
		size += SUMBIT(*(char*)sign);
		sign = (BITVECP) ( ((char*)sign) + 1 );
	);
	return size;	
}

Datum
gtxtidx_penalty(PG_FUNCTION_ARGS) {
	GISTENTRY *origentry = (GISTENTRY *)PG_GETARG_POINTER(0); /*always ISSIGNKEY*/
	GISTENTRY *newentry  = (GISTENTRY *)PG_GETARG_POINTER(1);
	float  *penalty = (float *)    PG_GETARG_POINTER(2);
	GISTTYPE *origval = (GISTTYPE*)DatumGetPointer( origentry->key );
	GISTTYPE *newval  = (GISTTYPE*)DatumGetPointer( newentry->key );
	int4 unionsize = 0;
	BITVECP	orig = GETSIGN(origval);

	if ( ISALLTRUE(origval) ) {
		*penalty = 0.0;
		PG_RETURN_POINTER( penalty );
	}

	if ( ISARRKEY(newval) ) {
		int4 *ptr=GETARR(newval), n=ARRNELEM(newval);
		while( n-- ) {
			if ( GETBIT(orig, HASHVAL( *ptr ) ) == 0 )
				unionsize++;
			ptr++;
		}
		*penalty = (float)unionsize;
	} else {
		if ( ISALLTRUE(newval) ) {
			*penalty = (float) (SIGLENBIT - sizebitvec( orig ) );
		} else {
			char valtmp;
			BITVECP nval = GETSIGN(newval);
			int4 i;

			LOOPBYTE(
				valtmp = nval[i] | orig[i];
				unionsize += SUMBIT(valtmp) - SUMBIT(orig[i]); 
			);
			*penalty = (float)unionsize;
		} 
	}

	PG_RETURN_POINTER( penalty );
}

typedef struct {
	bool	allistrue;
	BITVEC	sign;
} CACHESIGN;

static void
fillcache( CACHESIGN *item, GISTTYPE *key ) {
	item->allistrue = false;
	if ( ISARRKEY( key ) ) {
		makesign(item->sign, key);
	} else if ( ISALLTRUE(key) ) { 
		item->allistrue = true;
	} else {
		memcpy( (void*)item->sign, (void*)GETSIGN(key), sizeof(BITVEC));
	} 
} 

#define WISH_F(a,b,c) (double)( -(double)(((a)-(b))*((a)-(b))*((a)-(b)))*(c) )
typedef struct {
	OffsetNumber    pos;
	int4           cost;
} SPLITCOST;

static int
comparecost( const void *a, const void *b ) {
	if ( ((SPLITCOST*)a)->cost == ((SPLITCOST*)b)->cost )
		return 0;
	else
		return ( ((SPLITCOST*)a)->cost > ((SPLITCOST*)b)->cost ) ? 1 : -1;
}

Datum
gtxtidx_picksplit(PG_FUNCTION_ARGS) {
	bytea *entryvec = (bytea *)PG_GETARG_POINTER(0);
	GIST_SPLITVEC *v = (GIST_SPLITVEC *)PG_GETARG_POINTER(1);
	OffsetNumber k,j;
	GISTTYPE *datum_l, *datum_r;
	BITVEC union_l, union_r;
	bool            firsttime = true;
	int4	size_alpha,size_beta,sizeu,sizei;
	int4	size_waste, waste = 0.0;
	int4 	size_l, size_r;
	int4	nbytes;
	OffsetNumber seed_1=0, seed_2=0;
	OffsetNumber	*left, *right;
	OffsetNumber maxoff;
	BITVECP ptra, ptrb, ptrc;
	int i;
	CACHESIGN  *cache;
	char	valtmp;
	SPLITCOST	*costvector;

	maxoff = ((VARSIZE(entryvec) - VARHDRSZ) / sizeof(GISTENTRY)) - 2;
	nbytes = (maxoff + 2) * sizeof(OffsetNumber);
	v->spl_left = (OffsetNumber *) palloc(nbytes);
	v->spl_right = (OffsetNumber *) palloc(nbytes);

	cache = (CACHESIGN*)palloc(sizeof(CACHESIGN)*(maxoff+2));
	fillcache( &cache[FirstOffsetNumber], GETENTRY(entryvec,FirstOffsetNumber) ); 

	for (k = FirstOffsetNumber; k < maxoff; k = OffsetNumberNext(k)) {
		for (j = OffsetNumberNext(k); j <= maxoff; j = OffsetNumberNext(j)) {
			if ( k==FirstOffsetNumber )
				fillcache( &cache[j], GETENTRY(entryvec,j) );
			
			if ( cache[k].allistrue || cache[j].allistrue ) {
				sizeu = SIGLENBIT;
				if ( cache[k].allistrue && cache[j].allistrue )
					sizei = SIGLENBIT;
				else
					sizei = ( cache[k].allistrue ) ? 
						sizebitvec( cache[j].sign ) : sizebitvec( cache[k].sign );
			} else {
				sizeu = sizei = 0;
				ptra = cache[j].sign;
				ptrb = cache[k].sign;
				/* critical section for bench !!! */

#define COUNT(pos) do { \
	if ( GETBITBYTE(*(char*)ptra,pos) ) { \
		sizeu++; \
		if ( GETBITBYTE(*(char*)ptrb, pos) ) \
			sizei++; \
	} else if ( GETBITBYTE(*(char*)ptrb, pos) ) \
		sizeu++; \
} while(0)
				LOOPBYTE(
					COUNT(0);
					COUNT(1);
					COUNT(2);
					COUNT(3);
					COUNT(4);
					COUNT(5);
					COUNT(6);
					COUNT(7);
					ptra = (BITVECP) ( ((char*)ptra) + 1 );
					ptrb = (BITVECP) ( ((char*)ptrb) + 1 );
				);

			}
			size_waste = sizeu - sizei;
			if (size_waste > waste || firsttime) {
				waste = size_waste;
				seed_1 = k;
				seed_2 = j;
				firsttime = false;
			}
		}
	}

	left = v->spl_left;
	v->spl_nleft = 0;
	right = v->spl_right;
	v->spl_nright = 0;

	if ( seed_1 == 0 || seed_2 == 0 ) {
		seed_1 = 1;
		seed_2 = 2;
	}

	/* form initial .. */ 
	if ( cache[seed_1].allistrue ) {
		datum_l = (GISTTYPE*)palloc( CALCGTSIZE( SIGNKEY|ALLISTRUE, 0 ) );
		datum_l->len = CALCGTSIZE( SIGNKEY|ALLISTRUE, 0 ); datum_l->flag = SIGNKEY|ALLISTRUE;
		size_l = SIGLENBIT;
	} else {
		datum_l = (GISTTYPE*)palloc( CALCGTSIZE( SIGNKEY, 0 ) );
		datum_l->len = CALCGTSIZE( SIGNKEY, 0 ); datum_l->flag = SIGNKEY;
		memcpy((void*)GETSIGN(datum_l), (void*)cache[seed_1].sign, sizeof(BITVEC));
		size_l = sizebitvec( GETSIGN(datum_l) );		
	}
	if ( cache[seed_2].allistrue ) {
		datum_r = (GISTTYPE*)palloc( CALCGTSIZE( SIGNKEY|ALLISTRUE, 0 ) );
		datum_r->len = CALCGTSIZE( SIGNKEY|ALLISTRUE, 0 ); datum_r->flag = SIGNKEY|ALLISTRUE;
		size_r = SIGLENBIT;
	} else {
		datum_r = (GISTTYPE*)palloc( CALCGTSIZE( SIGNKEY, 0 ) );
		datum_r->len = CALCGTSIZE( SIGNKEY, 0 ); datum_r->flag = SIGNKEY;
		memcpy((void*)GETSIGN(datum_r), (void*)cache[seed_2].sign, sizeof(BITVEC));
		size_r = sizebitvec( GETSIGN(datum_r) );		
	}
	
	maxoff = OffsetNumberNext(maxoff);
	fillcache( &cache[maxoff], GETENTRY(entryvec,maxoff) ); 
	/* sort before ... */
	costvector=(SPLITCOST*)palloc( sizeof(SPLITCOST)*maxoff );
	for (j = FirstOffsetNumber; j <= maxoff; j = OffsetNumberNext(j)) {
		costvector[j-1].pos = j;
		if ( cache[j].allistrue ) {
			size_alpha = SIGLENBIT - size_l;
			size_beta  = SIGLENBIT - size_r;
		} else {
			ptra = cache[seed_1].sign;
			ptrb = cache[seed_2].sign;
			ptrc = cache[j].sign;
			size_beta = size_alpha = 0;
			if ( cache[seed_1].allistrue ) {
				if ( ! cache[seed_2].allistrue ) {
					LOOPBIT(
						if ( GETBIT(ptrc,i) && ! GETBIT(ptrb,i) ) 
							size_beta++;
					); 
				}
			} else if ( cache[seed_2].allistrue ) {
				if ( ! cache[seed_1].allistrue ) {
					LOOPBIT(
						if ( GETBIT(ptrc,i) && ! GETBIT(ptra,i) ) 
							size_alpha++;
					); 
				}
			} else {
				LOOPBIT(
					if ( GETBIT(ptrc,i) && ! GETBIT(ptra,i) ) 
						size_alpha++; 
					if ( GETBIT(ptrc,i) && ! GETBIT(ptrb,i) ) 
						size_beta++; 
				);
			}

		}
		costvector[j-1].cost = abs( size_alpha - size_beta ); 
	}
	qsort( (void*)costvector, maxoff, sizeof(SPLITCOST), comparecost );	
	
	for (k = 0; k < maxoff; k++) {
		j = costvector[k].pos;
		if ( j == seed_1 ) {
			*left++ = j;
			v->spl_nleft++;
			continue;
		} else if ( j == seed_2 ) {
			*right++ = j;
			v->spl_nright++;
			continue;
		}

		if ( ISALLTRUE( datum_l ) || cache[j].allistrue ) {
			size_alpha = SIGLENBIT;
		} else {
			ptra = cache[j].sign;
			ptrb = GETSIGN(datum_l);
			size_alpha = 0;
			LOOPBYTE(
				valtmp = union_l[i] = ptra[i] | ptrb[i];
				size_alpha += SUMBIT( valtmp );
			);
		}
		if ( ISALLTRUE( datum_r ) || cache[j].allistrue ) {
			size_beta = SIGLENBIT;
		} else {
			ptra = cache[j].sign;
			ptrb = GETSIGN(datum_r);
			size_beta = 0;
			LOOPBYTE(
				valtmp = union_r[i] = ptra[i] | ptrb[i];
				size_beta += SUMBIT( valtmp );
			);
		}

		if (size_alpha - size_l < size_beta - size_r + WISH_F(v->spl_nleft, v->spl_nright, 0.1)) {
			if ( ! ISALLTRUE( datum_l ) ) {
				if ( size_alpha == SIGLENBIT ) {
					if ( size_alpha != size_l ) 
						MemSet( (void*)GETSIGN(datum_l),0xff, sizeof(BITVEC));
				} else 
					memcpy( (void*)GETSIGN(datum_l), (void*)union_l, sizeof(BITVEC) );
			}
			size_l = size_alpha;
			*left++ = j;
			v->spl_nleft++;
		} else {
			if ( ! ISALLTRUE( datum_r ) ) {
				if ( size_beta == SIGLENBIT ) {
					if ( size_beta != size_r ) 
						MemSet( (void*)GETSIGN(datum_r),0xff, sizeof(BITVEC));
				} else 
					memcpy( (void*)GETSIGN(datum_r), (void*)union_r, sizeof(BITVEC) );
			}
			size_r = size_beta;
			*right++ = j;
			v->spl_nright++;
		}
	}
	
	*right = *left = FirstOffsetNumber;
	pfree(costvector);
	pfree(cache);
	v->spl_ldatum = PointerGetDatum(datum_l);
	v->spl_rdatum = PointerGetDatum(datum_r);

	PG_RETURN_POINTER( v );
}