Merge remote-tracking branch 'origin/mdb.master'

libraries/liblmdb/Makefile

@@ -1,3 +1,34 @@
+# Makefile for liblmdb (Lightning memory-mapped database library).
+
+########################################################################
+# Configuration. The compiler options must enable threaded compilation.
+#
+# Preprocessor macros (for CPPFLAGS) of interest:
+#
+# To compile successfully if the default does not:
+# - MDB_USE_POSIX_SEM	(enabled by default on BSD, Apple)
+#	Define if shared mutexes are unsupported.  Note that Posix
+#	semaphores and shared mutexes have different behaviors and
+#	different problems, see the Caveats section in lmdb.h.
+#
+# For best performence or to compile successfully:
+# - MDB_DSYNC = "O_DSYNC" (default) or "O_SYNC" (less efficient)
+#	If O_DSYNC is undefined but exists in /usr/include,
+#	preferably set some compiler flag to get the definition.
+# - MDB_FDATASYNC = "fdatasync" or "fsync"
+#	Function for flushing the data of a file. Define this to
+#	"fsync" if fdatasync() is not supported. fdatasync is
+#	default except on BSD, Apple, Android which use fsync.
+# - MDB_USE_PWRITEV
+#	Define if the pwritev() function is supported.
+#
+# Data format:
+# - MDB_MAXKEYSIZE
+#	Controls data packing and limits, see mdb.c.
+#
+# Debugging:
+# - MDB_DEBUG, MDB_PARANOID.
+#
 CC	= gcc
 W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast
 OPT = -O2 -g
@@ -6,6 +37,8 @@ LDLIBS	=
 SOLIBS	=
 prefix	= /usr/local
 
+########################################################################
+
 IHDRS	= lmdb.h
 ILIBS	= liblmdb.a liblmdb.so
 IPROGS	= mdb_stat mdb_copy

libraries/liblmdb/lmdb.h

@@ -144,6 +144,16 @@ typedef	int	mdb_mode_t;
 typedef	mode_t	mdb_mode_t;
 #endif
 
+/** An abstraction for a file handle.
+ *	On POSIX systems file handles are small integers. On Windows
+ *	they're opaque pointers.
+ */
+#ifdef _WIN32
+typedef	void *mdb_filehandle_t;
+#else
+typedef int mdb_filehandle_t;
+#endif
+
 /** @defgroup mdb MDB API
  *	@{
  *	@brief OpenLDAP Lightning Memory-Mapped Database Manager
@@ -325,13 +335,11 @@ typedef enum MDB_cursor_op {
 								Only for #MDB_DUPSORT */
 	MDB_NEXT_MULTIPLE,		/**< Return all duplicate data items at the next
 								cursor position. Only for #MDB_DUPFIXED */
-	MDB_NEXT_NODUP,			/**< Position at first data item of next key.
+	MDB_NEXT_NODUP,			/**< Position at first data item of next key */
-								Only for #MDB_DUPSORT */
 	MDB_PREV,				/**< Position at previous data item */
 	MDB_PREV_DUP,			/**< Position at previous data item of current key.
 								Only for #MDB_DUPSORT */
-	MDB_PREV_NODUP,			/**< Position at last data item of previous key.
+	MDB_PREV_NODUP,			/**< Position at last data item of previous key */
-								Only for #MDB_DUPSORT */
 	MDB_SET,				/**< Position at specified key */
 	MDB_SET_KEY,			/**< Position at specified key, return key + data */
 	MDB_SET_RANGE			/**< Position at first key greater than or equal to specified key. */
@@ -535,6 +543,17 @@ int  mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t
 	 */
 int  mdb_env_copy(MDB_env *env, const char *path);
 
+	/** @brief Copy an MDB environment to the specified file descriptor.
+	 *
+	 * This function may be used to make a backup of an existing environment.
+	 * @param[in] env An environment handle returned by #mdb_env_create(). It
+	 * must have already been opened successfully.
+	 * @param[in] fd The filedescriptor to write the copy to. It must
+	 * have already been opened for Write access.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int  mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
+
 	/** @brief Return statistics about the MDB environment.
 	 *
 	 * @param[in] env An environment handle returned by #mdb_env_create()

libraries/liblmdb/mdb_copy.1

@@ -5,12 +5,19 @@
 mdb_copy \- LMDB environment copy tool
 .SH SYNOPSIS
 .B mdb_copy
-.I srcpath\ dstpath
+.I srcpath\ [dstpath]
 .SH DESCRIPTION
 The
 .B mdb_copy
 utility copies an LMDB environment. The environment can
 be copied regardless of whether it is currently in use.
+
+If
+.I dstpath
+is specified it must be the path of an empty directory
+for storing the backup. Otherwise, the backup will be
+written to stdout.
+
 .SH DIAGNOSTICS
 Exit status is zero if no errors occur.
 Errors result in a non-zero exit status and

libraries/liblmdb/mdb_copy.c

@@ -11,28 +11,52 @@
  * top-level directory of the distribution or, alternatively, at
  * <http://www.OpenLDAP.org/license.html>.
  */
+#ifdef _WIN32
+#include <windows.h>
+#define	MDB_STDOUT	GetStdHandle(STD_OUTPUT_HANDLE)
+#else
+#define	MDB_STDOUT	1
+#endif
 #include <stdio.h>
 #include <stdlib.h>
+#include <signal.h>
 #include "lmdb.h"
 
+static void
+sighandle(int sig)
+{
+}
+
 int main(int argc,char * argv[])
 {
 	int rc;
 	MDB_env *env;
 	char *envname = argv[1];
 
-	if (argc != 3) {
+	if (argc<2 || argc>3) {
-		fprintf(stderr, "usage: %s srcpath dstpath\n", argv[0]);
+		fprintf(stderr, "usage: %s srcpath [dstpath]\n", argv[0]);
 		exit(EXIT_FAILURE);
 	}
 
+#ifdef SIGPIPE
+	signal(SIGPIPE, sighandle);
+#endif
+#ifdef SIGHUP
+	signal(SIGHUP, sighandle);
+#endif
+	signal(SIGINT, sighandle);
+	signal(SIGTERM, sighandle);
+
 	rc = mdb_env_create(&env);
 
 	rc = mdb_env_open(env, envname, MDB_RDONLY, 0);
 	if (rc) {
 		printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 	} else {
-		rc = mdb_env_copy(env, argv[2]);
+		if (argc == 2)
+			rc = mdb_env_copyfd(env, MDB_STDOUT);
+		else
+			rc = mdb_env_copy(env, argv[2]);
 		if (rc)
 			printf("mdb_env_copy failed, error %d %s\n", rc, mdb_strerror(rc));
 	}

libraries/liblmdb/mdb_stat.c

@@ -193,9 +193,12 @@ int main(int argc, char *argv[])
 			printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 			goto txn_abort;
 		}
-		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT)) == 0) {
+		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
-			char *str = malloc(key.mv_size+1);
+			char *str;
 			MDB_dbi db2;
+			if (memchr(key.mv_data, '\0', key.mv_size))
+				continue;
+			str = malloc(key.mv_size+1);
 			memcpy(str, key.mv_data, key.mv_size);
 			str[key.mv_size] = '\0';
 			rc = mdb_open(txn, str, 0, &db2);
@@ -214,6 +217,9 @@ int main(int argc, char *argv[])
 		mdb_cursor_close(cursor);
 	}
 
+	if (rc == MDB_NOTFOUND)
+		rc = MDB_SUCCESS;
+
 	mdb_close(env, dbi);
 txn_abort:
 	mdb_txn_abort(txn);

libraries/liblmdb/midl.c

@@ -71,17 +71,6 @@ int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
 {
 	unsigned x, i;
 
-	if (MDB_IDL_IS_RANGE( ids )) {
-		/* if already in range, treat as a dup */
-		if (id >= MDB_IDL_RANGE_FIRST(ids) && id <= MDB_IDL_RANGE_LAST(ids))
-			return -1;
-		if (id < MDB_IDL_RANGE_FIRST(ids))
-			ids[1] = id;
-		else if (id > MDB_IDL_RANGE_LAST(ids))
-			ids[2] = id;
-		return 0;
-	}
-
 	x = mdb_midl_search( ids, id );
 	assert( x > 0 );
 
@@ -97,15 +86,9 @@ int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
 	}
 
 	if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
-		if( id < ids[1] ) {
+		/* no room */
-			ids[1] = id;
+		--ids[0];
-			ids[2] = ids[ids[0]-1];
+		return -2;
-		} else if ( ids[ids[0]-1] < id ) {
-			ids[2] = id;
-		} else {
-			ids[2] = ids[ids[0]-1];
-		}
-		ids[0] = MDB_NOID;
 	
 	} else {
 		/* insert id */
@@ -121,8 +104,10 @@ int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
 MDB_IDL mdb_midl_alloc(int num)
 {
 	MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
-	if (ids)
+	if (ids) {
 		*ids++ = num;
+		*ids = 0;
+	}
 	return ids;
 }
 
@@ -135,8 +120,9 @@ void mdb_midl_free(MDB_IDL ids)
 int mdb_midl_shrink( MDB_IDL *idp )
 {
 	MDB_IDL ids = *idp;
-	if (*(--ids) > MDB_IDL_UM_MAX) {
+	if (*(--ids) > MDB_IDL_UM_MAX &&
-		ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID));
+		(ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID))))
+	{
 		*ids++ = MDB_IDL_UM_MAX;
 		*idp = ids;
 		return 1;
@@ -144,7 +130,7 @@ int mdb_midl_shrink( MDB_IDL *idp )
 	return 0;
 }
 
-int mdb_midl_grow( MDB_IDL *idp, int num )
+static int mdb_midl_grow( MDB_IDL *idp, int num )
 {
 	MDB_IDL idn = *idp-1;
 	/* grow it */
@@ -156,6 +142,20 @@ int mdb_midl_grow( MDB_IDL *idp, int num )
 	return 0;
 }
 
+int mdb_midl_need( MDB_IDL *idp, unsigned num )
+{
+	MDB_IDL ids = *idp;
+	num += ids[0];
+	if (num > ids[-1]) {
+		num = (num + num/4 + (256 + 2)) & -256;
+		if (!(ids = realloc(ids-1, num * sizeof(MDB_ID))))
+			return ENOMEM;
+		*ids++ = num -= 2;
+		*idp = ids;
+	}
+	return 0;
+}
+
 int mdb_midl_append( MDB_IDL *idp, MDB_ID id )
 {
 	MDB_IDL ids = *idp;
@@ -184,6 +184,22 @@ int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app )
 	return 0;
 }
 
+int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n )
+{
+	MDB_ID *ids = *idp, len = ids[0];
+	/* Too big? */
+	if (len + n > ids[-1]) {
+		if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
+			return ENOMEM;
+		ids = *idp;
+	}
+	ids[0] = len + n;
+	ids += len;
+	while (n)
+		ids[n--] = id++;
+	return 0;
+}
+
 /* Quicksort + Insertion sort for small arrays */
 
 #define SMALL	8

libraries/liblmdb/midl.h

@@ -52,64 +52,33 @@ typedef size_t MDB_ID;
 	 */
 typedef MDB_ID *MDB_IDL;
 
-#define	MDB_NOID	(~(MDB_ID)0)
-
 /* IDL sizes - likely should be even bigger
  *   limiting factors: sizeof(ID), thread stack size
  */
 #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
 #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))
-#define MDB_IDL_UM_SIZEOF	(MDB_IDL_UM_SIZE * sizeof(MDB_ID))
 
 #define MDB_IDL_DB_MAX		(MDB_IDL_DB_SIZE-1)
-
 #define MDB_IDL_UM_MAX		(MDB_IDL_UM_SIZE-1)
 
-#define MDB_IDL_IS_RANGE(ids)	((ids)[0] == MDB_NOID)
+#define MDB_IDL_SIZEOF(ids)		(((ids)[0]+1) * sizeof(MDB_ID))
-#define MDB_IDL_RANGE_SIZE		(3)
-#define MDB_IDL_RANGE_SIZEOF	(MDB_IDL_RANGE_SIZE * sizeof(MDB_ID))
-#define MDB_IDL_SIZEOF(ids)		((MDB_IDL_IS_RANGE(ids) \
-	? MDB_IDL_RANGE_SIZE : ((ids)[0]+1)) * sizeof(MDB_ID))
-
-#define MDB_IDL_RANGE_FIRST(ids)	((ids)[1])
-#define MDB_IDL_RANGE_LAST(ids)		((ids)[2])
-
-#define MDB_IDL_RANGE( ids, f, l ) \
-	do { \
-		(ids)[0] = MDB_NOID; \
-		(ids)[1] = (f);  \
-		(ids)[2] = (l);  \
-	} while(0)
-
-#define MDB_IDL_ZERO(ids) \
-	do { \
-		(ids)[0] = 0; \
-		(ids)[1] = 0; \
-		(ids)[2] = 0; \
-	} while(0)
-
 #define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 )
-#define MDB_IDL_IS_ALL( range, ids ) ( (ids)[0] == MDB_NOID \
-	&& (ids)[1] <= (range)[1] && (range)[2] <= (ids)[2] )
-
 #define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) ))
-
-#define MDB_IDL_ID( bdb, ids, id ) MDB_IDL_RANGE( ids, id, ((bdb)->bi_lastid) )
-#define MDB_IDL_ALL( bdb, ids ) MDB_IDL_RANGE( ids, 1, ((bdb)->bi_lastid) )
-
 #define MDB_IDL_FIRST( ids )	( (ids)[1] )
-#define MDB_IDL_LAST( ids )		( MDB_IDL_IS_RANGE(ids) \
+#define MDB_IDL_LAST( ids )		( (ids)[(ids)[0]] )
-	? (ids)[2] : (ids)[(ids)[0]] )
 
-#define MDB_IDL_N( ids )		( MDB_IDL_IS_RANGE(ids) \
+	/** Append ID to IDL. The IDL must be big enough. */
-	? ((ids)[2]-(ids)[1])+1 : (ids)[0] )
+#define mdb_midl_xappend(idl, id) do { \
+		MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \
+		xidl[xlen] = (id); \
+	} while (0)
 
 #if 0	/* superseded by append/sort */
 	/** Insert an ID into an IDL.
 	 * @param[in,out] ids	The IDL to insert into.
 	 * @param[in] id	The ID to insert.
-	 * @return	0 on success, -1 if the ID was already present in the IDL.
+	 * @return	0 on success, -1 if ID was already present, -2 on error.
 	 */
 int mdb_midl_insert( MDB_IDL ids, MDB_ID id );
 #endif
@@ -132,28 +101,35 @@ void mdb_midl_free(MDB_IDL ids);
 	 */
 int mdb_midl_shrink(MDB_IDL *idp);
 
-	/** Grow an IDL.
+	/** Make room for num additional elements in an IDL.
-	 * Add room for num additional elements.
+	 * @param[in,out] idp	Address of the IDL.
-	 * @param[in,out] idp	Address of the IDL to grow.
+	 * @param[in] num	Number of elements to make room for.
-	 * @param[in] num	Number of elements to add.
+	 * @return	0 on success, ENOMEM on failure.
-	 * @return	0 on success, -1 on failure.
 	 */
-int mdb_midl_grow(MDB_IDL *idp, int num);
+int mdb_midl_need(MDB_IDL *idp, unsigned num);
 
 	/** Append an ID onto an IDL.
 	 * @param[in,out] idp	Address of the IDL to append to.
 	 * @param[in] id	The ID to append.
-	 * @return	0 on success, -1 if the IDL is too large.
+	 * @return	0 on success, ENOMEM if the IDL is too large.
 	 */
 int mdb_midl_append( MDB_IDL *idp, MDB_ID id );
 
 	/** Append an IDL onto an IDL.
 	 * @param[in,out] idp	Address of the IDL to append to.
 	 * @param[in] app	The IDL to append.
-	 * @return	0 on success, -1 if the IDL is too large.
+	 * @return	0 on success, ENOMEM if the IDL is too large.
 	 */
 int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app );
 
+	/** Append an ID range onto an IDL.
+	 * @param[in,out] idp	Address of the IDL to append to.
+	 * @param[in] id	The lowest ID to append.
+	 * @param[in] n		Number of IDs to append.
+	 * @return	0 on success, ENOMEM if the IDL is too large.
+	 */
+int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n );
+
 	/** Sort an IDL.
 	 * @param[in,out] ids	The IDL to sort.
 	 */