diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index b0aad290ae..cc88d8c8e6 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -20,8 +20,11 @@ LMDB 0.9.17 Release Engineering Fix ITS#8312 loose pages in nested txn Fix ITS#8313 mdb_rebalance dummy cursor Fix ITS#8315 dirty_room in nested txn + Fix ITS#8323 dirty_list in nested txn Fix ITS#8316 page_merge cursor tracking Fix ITS#8321 cursor tracking + Fix ITS#8319 mdb_load error messages + Fix ITS#8320 mdb_load plaintext input Added mdb_txn_id() (ITS#7994) Added robust mutex support Miscellaneous cleanup/simplification @@ -31,6 +34,8 @@ LMDB 0.9.17 Release Engineering Added ssize_t typedef for MSVC (ITS#8067) Use ANSI apis on Windows (ITS#8069) Use O_SYNC if O_DSYNC,MDB_DSYNC are not defined (ITS#7209) + Allow passing AR to make (ITS#8168) + Allow passing mandir to make install (ITS#8169) LMDB 0.9.16 Release (2015/08/14) Fix cursor EOF bug (ITS#8190) diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index 79752d2995..dbb5d698fe 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -19,6 +19,7 @@ # read mdb.c before changing any of them. # CC = gcc +AR = ar W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized THREADS = -pthread OPT = -O2 -g @@ -26,6 +27,7 @@ CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) LDLIBS = SOLIBS = prefix = /usr/local +mandir = $(prefix)/man ######################################################################## @@ -44,7 +46,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS) for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done - for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done + for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done clean: rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb @@ -54,7 +56,7 @@ test: all ./mtest && ./mdb_stat testdb liblmdb.a: mdb.o midl.o - ar rs $@ mdb.o midl.o + $(AR) rs $@ mdb.o midl.o liblmdb.so: mdb.lo midl.lo # $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index c5174e983a..b87099e487 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1613,6 +1613,13 @@ mdb_cursor_chk(MDB_cursor *mc) } if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) printf("ack!\n"); + if (mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && + mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { + printf("blah!\n"); + } + } } #endif @@ -2418,14 +2425,15 @@ done: } else { for (; m2; m2=m2->mc_next) { if (m2->mc_snum < mc->mc_snum) continue; + if (m2 == mc) continue; if (m2->mc_pg[mc->mc_top] == mp) { m2->mc_pg[mc->mc_top] = np; if ((mc->mc_db->md_flags & MDB_DUPSORT) && IS_LEAF(np) && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + (m2->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); - if (!(leaf->mn_flags & F_SUBDATA)) + MDB_node *leaf = NODEPTR(np, m2->mc_ki[mc->mc_top]); + if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); } } @@ -3431,6 +3439,25 @@ mdb_txn_commit(MDB_txn *txn) pspill[0] = y; } + /* Remove anything in our spill list from parent's dirty list */ + if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) { + for (i=1; i<=txn->mt_spill_pgs[0]; i++) { + MDB_ID pn = txn->mt_spill_pgs[i]; + if (pn & 1) + continue; /* deleted spillpg */ + pn >>= 1; + y = mdb_mid2l_search(dst, pn); + if (y <= dst[0].mid && dst[y].mid == pn) { + free(dst[y].mptr); + while (y < dst[0].mid) { + dst[y] = dst[y+1]; + y++; + } + dst[0].mid--; + } + } + } + /* Find len = length of merging our dirty list with parent's */ x = dst[0].mid; dst[0].mid = 0; /* simplify loops */ @@ -5144,8 +5171,11 @@ mdb_cursor_pop(MDB_cursor *mc) mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); mc->mc_snum--; - if (mc->mc_snum) + if (mc->mc_snum) { mc->mc_top--; + } else { + mc->mc_flags &= ~C_INITIALIZED; + } } } @@ -5856,6 +5886,8 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, } else return MDB_NOTFOUND; } + } else { + mc->mc_pg[0] = 0; } rc = mdb_page_search(mc, key, 0); @@ -6073,8 +6105,6 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_GET_KEY(leaf, key); if (data) { if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (mc->mc_flags & C_DEL) - mdb_xcursor_init1(mc, leaf); rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); } else { rc = mdb_node_read(mc->mc_txn, leaf, data); @@ -6653,22 +6683,28 @@ new_sub: } else { /* There is room already in this leaf page. */ rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); - if (rc == 0 && insert_key) { + if (rc == 0) { /* Adjust other cursors pointing to mp */ MDB_cursor *m2, *m3; MDB_dbi dbi = mc->mc_dbi; unsigned i = mc->mc_top; MDB_page *mp = mc->mc_pg[i]; + int nkeys = NUMKEYS(mp); for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { if (mc->mc_flags & C_SUB) m3 = &m2->mc_xcursor->mx_cursor; else m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { + if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue; + if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { m3->mc_ki[i]++; } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + MDB_node *n2 = NODEPTR(mp, m3->mc_ki[i]); + if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } } } } @@ -6801,6 +6837,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) if (flags & MDB_NODUPDATA) { /* mdb_cursor_del0() will subtract the final entry */ mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; } else { if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); @@ -6836,8 +6873,9 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) } } mc->mc_db->md_entries--; - mc->mc_flags |= C_DEL; return rc; + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; } /* otherwise fall thru and delete the sub-DB */ } @@ -7524,21 +7562,21 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key) static void mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); -/** Track a temporary cursor */ -#define CURSOR_TMP_TRACK(mc, mn, dummy, tracked) \ - if (mc->mc_flags & C_SUB) { \ +/** Perform \b act while tracking temporary cursor \b mn */ +#define WITH_CURSOR_TRACKING(mn, act) do { \ + MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ + if ((mn).mc_flags & C_SUB) { \ dummy.mc_flags = C_INITIALIZED; \ - dummy.mc_xcursor = (MDB_xcursor *)&mn; \ + dummy.mc_xcursor = (MDB_xcursor *)&(mn); \ tracked = &dummy; \ } else { \ - tracked = &mn; \ + tracked = &(mn); \ } \ - tracked->mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; \ - mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked - -/** Stop tracking a temporary cursor */ -#define CURSOR_TMP_UNTRACK(mc, tracked) \ - mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked->mc_next + tracked->mc_next = *tp; \ + *tp = tracked; \ + { act; } \ + *tp = tracked->mc_next; \ +} while (0) /** Move a node from csrc to cdst. */ @@ -7595,6 +7633,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) data.mv_size = NODEDSZ(srcnode); data.mv_data = NODEDATA(srcnode); } + mn.mc_xcursor = NULL; if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { unsigned int snum = cdst->mc_snum; MDB_node *s2; @@ -7666,6 +7705,12 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; m3->mc_ki[csrc->mc_top-1]++; } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && + IS_LEAF(mps)) { + MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } } } else /* Adding on the right, bump others down */ @@ -7686,6 +7731,12 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) } else { m3->mc_ki[csrc->mc_top]--; } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && + IS_LEAF(mps)) { + MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } } } } @@ -7695,7 +7746,6 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) */ if (csrc->mc_ki[csrc->mc_top] == 0) { if (csrc->mc_ki[csrc->mc_top-1] != 0) { - MDB_cursor dummy, *tracked; if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); } else { @@ -7709,9 +7759,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) mn.mc_snum--; mn.mc_top--; /* We want mdb_rebalance to find mn when doing fixups */ - CURSOR_TMP_TRACK(csrc, mn, dummy, tracked); - rc = mdb_update_key(&mn, &key); - CURSOR_TMP_UNTRACK(csrc, tracked); + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); if (rc) return rc; } @@ -7728,7 +7777,6 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) if (cdst->mc_ki[cdst->mc_top] == 0) { if (cdst->mc_ki[cdst->mc_top-1] != 0) { - MDB_cursor dummy, *tracked; if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); } else { @@ -7742,9 +7790,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) mn.mc_snum--; mn.mc_top--; /* We want mdb_rebalance to find mn when doing fixups */ - CURSOR_TMP_TRACK(cdst, mn, dummy, tracked); - rc = mdb_update_key(&mn, &key); - CURSOR_TMP_UNTRACK(cdst, tracked); + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); if (rc) return rc; } @@ -7814,6 +7861,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) MDB_cursor mn; MDB_node *s2; mdb_cursor_copy(csrc, &mn); + mn.mc_xcursor = NULL; /* must find the lowest key below src */ rc = mdb_page_search_lowest(&mn); if (rc) @@ -7889,6 +7937,12 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { m3->mc_ki[top-1]--; } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && + IS_LEAF(psrc)) { + MDB_node *node = NODEPTR(m3->mc_pg[top], m3->mc_ki[top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } } } { @@ -8103,13 +8157,11 @@ mdb_rebalance(MDB_cursor *mc) if (!fromleft) { rc = mdb_page_merge(&mn, mc); } else { - MDB_cursor dummy, *tracked; oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; /* We want mdb_rebalance to find mn when doing fixups */ - CURSOR_TMP_TRACK(mc, mn, dummy, tracked); - rc = mdb_page_merge(mc, &mn); - CURSOR_TMP_UNTRACK(mc, tracked); + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_merge(mc, &mn)); mdb_cursor_copy(&mn, mc); } mc->mc_flags &= ~C_EOF; @@ -8142,12 +8194,17 @@ mdb_cursor_del0(MDB_cursor *mc) if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= ki) { + if (m3->mc_ki[mc->mc_top] == ki) { m3->mc_flags |= C_DEL; - if (m3->mc_ki[mc->mc_top] > ki) - m3->mc_ki[mc->mc_top]--; - else if (mc->mc_db->md_flags & MDB_DUPSORT) + if (mc->mc_db->md_flags & MDB_DUPSORT) m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } else if (m3->mc_ki[mc->mc_top] > ki) { + m3->mc_ki[mc->mc_top]--; + } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); } } } @@ -8331,6 +8388,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno } mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; mn.mc_pg[mn.mc_top] = rp; mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; @@ -8473,14 +8531,12 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno */ if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { int snum = mc->mc_snum; - MDB_cursor dummy, *tracked; mn.mc_snum--; mn.mc_top--; did_split = 1; /* We want other splits to find mn when doing fixups */ - CURSOR_TMP_TRACK(mc, mn, dummy, tracked); - rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); - CURSOR_TMP_UNTRACK(mc, tracked); + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0)); if (rc) goto done; @@ -8639,7 +8695,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno m3->mc_ki[k+1] = m3->mc_ki[k]; m3->mc_pg[k+1] = m3->mc_pg[k]; } - if (m3->mc_ki[0] > nkeys) { + if (m3->mc_ki[0] >= nkeys) { m3->mc_ki[0] = 1; } else { m3->mc_ki[0] = 0; @@ -8663,6 +8719,12 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { m3->mc_ki[ptop]++; } + if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && + IS_LEAF(mp)) { + MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } } } DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); @@ -8681,6 +8743,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, { MDB_cursor mc; MDB_xcursor mx; + int rc; if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) return EINVAL; @@ -8692,7 +8755,11 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; mdb_cursor_init(&mc, txn, dbi, &mx); - return mdb_cursor_put(&mc, key, data, flags); + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_put(&mc, key, data, flags); + txn->mt_cursors[dbi] = mc.mc_next; + return rc; } #ifndef MDB_WBUF @@ -9635,6 +9702,7 @@ done: } else if (rc == MDB_NOTFOUND) { rc = MDB_SUCCESS; } + mc->mc_flags &= ~C_INITIALIZED; return rc; } diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 160bc1b885..d2f0968188 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -327,7 +327,7 @@ int main(int argc, char *argv[]) putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; break; case 'T': - mode |= NOHDR; + mode |= NOHDR | PRINT; break; default: usage(); @@ -400,20 +400,22 @@ int main(int argc, char *argv[]) while(1) { rc = readline(&key, &kbuf); - if (rc == EOF) + if (rc) /* rc == EOF */ break; - if (rc) - goto txn_abort; rc = readline(&data, &dbuf); - if (rc) + if (rc) { + fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno); goto txn_abort; - + } + rc = mdb_cursor_put(mc, &key, &data, putflags); if (rc == MDB_KEYEXIST && putflags) continue; - if (rc) + if (rc) { + fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; + } batch++; if (batch == 100) { rc = mdb_txn_commit(txn);